🌐 AI搜索 & 代理 主页
Skip to content

Commit ad06db2

Browse files
bmomjianbmomjian
committed
In pg_upgrade, use cached copy of directory listing, rather than calling
scandir() with a pattern for every table. Optimization after report of pg_upgrade slowness with 150k tables.
1 parent 07456b4 commit ad06db2

File tree

3 files changed

+74
-54
lines changed

3 files changed

+74
-54
lines changed

‎contrib/pg_upgrade/info.c‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ static void map_rel_by_id(Oid oldid, Oid newid,
3030
const char *old_tablespace, const DbInfo *old_db,
3131
const DbInfo *new_db, const char *olddata,
3232
const char *newdata, FileNameMap *map);
33-
static RelInfo *relarr_lookup_reloid(
34-
RelInfoArr *rel_arr, Oid oid, Cluster whichCluster);
33+
static RelInfo *relarr_lookup_reloid(RelInfoArr *rel_arr,
34+
Oid oid, Cluster whichCluster);
3535
static RelInfo *relarr_lookup_rel(RelInfoArr *rel_arr,
3636
const char *nspname, const char *relname,
3737
Cluster whichCluster);
@@ -172,18 +172,18 @@ map_rel_by_id(Oid oldid, Oid newid,
172172
* relation belongs to the default tablespace, hence relfiles would
173173
* exist in the data directories.
174174
*/
175-
snprintf(map->old_file, sizeof(map->old_file), "%s/base/%u", olddata, old_db->db_oid);
176-
snprintf(map->new_file, sizeof(map->new_file), "%s/base/%u", newdata, new_db->db_oid);
175+
snprintf(map->old_dir, sizeof(map->old_dir), "%s/base/%u", olddata, old_db->db_oid);
176+
snprintf(map->new_dir, sizeof(map->new_dir), "%s/base/%u", newdata, new_db->db_oid);
177177
}
178178
else
179179
{
180180
/*
181181
* relation belongs to some tablespace, hence copy its physical
182182
* location
183183
*/
184-
snprintf(map->old_file, sizeof(map->old_file), "%s%s/%u", old_tablespace,
184+
snprintf(map->old_dir, sizeof(map->old_dir), "%s%s/%u", old_tablespace,
185185
old_cluster.tablespace_suffix, old_db->db_oid);
186-
snprintf(map->new_file, sizeof(map->new_file), "%s%s/%u", old_tablespace,
186+
snprintf(map->new_dir, sizeof(map->new_dir), "%s%s/%u", old_tablespace,
187187
new_cluster.tablespace_suffix, new_db->db_oid);
188188
}
189189
}
@@ -318,6 +318,7 @@ get_rel_infos(const DbInfo *dbinfo,
318318
* pg_largeobject_loid_pn_index's relfilenode can change if the table was
319319
* reindexed so we get the relfilenode for each database and migrate it as
320320
* a normal user table.
321+
* Order by tablespace so we can cache the directory contents efficiently.
321322
*/
322323

323324
snprintf(query, sizeof(query),
@@ -338,7 +339,7 @@ get_rel_infos(const DbInfo *dbinfo,
338339
"GROUP BY c.oid, n.nspname, c.relname, c.relfilenode,"
339340
" c.reltoastrelid, t.spclocation, "
340341
" n.nspname "
341-
"ORDER BY n.nspname, c.relname;",
342+
"ORDER BY t.spclocation, n.nspname, c.relname;",
342343
FirstNormalObjectId,
343344
/* see the comment at the top of old_8_3_create_sequence_script() */
344345
(GET_MAJOR_VERSION(old_cluster.major_version) <= 803) ?

‎contrib/pg_upgrade/pg_upgrade.h‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@ typedef struct
8585
*/
8686
typedef struct
8787
{
88+
char old_dir[MAXPGPATH];
89+
char new_dir[MAXPGPATH];
8890
Oid old_relfilenode; /* Relfilenode of the old relation */
8991
Oid new_relfilenode; /* Relfilenode of the new relation */
90-
char old_file[MAXPGPATH];
91-
char new_file[MAXPGPATH];
9292
char old_nspname[NAMEDATALEN]; /* old name of the namespace */
9393
char old_relname[NAMEDATALEN]; /* old name of the relation */
9494
char new_nspname[NAMEDATALEN]; /* new name of the namespace */
@@ -255,10 +255,8 @@ void check_old_cluster(bool live_check,
255255
char **sequence_script_file_name);
256256
void check_new_cluster(void);
257257
void report_clusters_compatible(void);
258-
void issue_warnings(
259-
char *sequence_script_file_name);
260-
void output_completion_banner(
261-
char *deletion_script_file_name);
258+
void issue_warnings(char *sequence_script_file_name);
259+
void output_completion_banner(char *deletion_script_file_name);
262260
void check_cluster_versions(void);
263261
void check_cluster_compatibility(bool live_check);
264262
void create_script_for_old_cluster_deletion(char **deletion_script_file_name);
@@ -319,14 +317,12 @@ typedef void *pageCnvCtx;
319317
#endif
320318

321319
int dir_matching_filenames(const struct dirent * scan_ent);
322-
int pg_scandir(const char *dirname,
323-
struct dirent *** namelist,
324-
int (*selector) (const struct dirent *));
325-
const char *copyAndUpdateFile(
326-
pageCnvCtx *pageConverter, const char *src,
320+
int pg_scandir(const char *dirname, struct dirent *** namelist,
321+
int (*selector) (const struct dirent *));
322+
const char *copyAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
327323
const char *dst, bool force);
328-
const char *linkAndUpdateFile(
329-
pageCnvCtx *pageConverter, const char *src, const char *dst);
324+
const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src,
325+
const char *dst);
330326

331327
void check_hard_link(void);
332328

@@ -374,8 +370,7 @@ PGresult *executeQueryOrDie(PGconn *conn,
374370

375371
void start_postmaster(Cluster whichCluster, bool quiet);
376372
void stop_postmaster(bool fast, bool quiet);
377-
uint32 get_major_server_version(char **verstr,
378-
Cluster whichCluster);
373+
uint32 get_major_server_version(char **verstr, Cluster whichCluster);
379374
void check_for_libpq_envvars(void);
380375

381376

‎contrib/pg_upgrade/relfilenode.c‎

Lines changed: 56 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -110,20 +110,38 @@ static void
110110
transfer_single_new_db(pageCnvCtx *pageConverter,
111111
FileNameMap *maps, int size)
112112
{
113+
char old_dir[MAXPGPATH];
114+
struct dirent **namelist = NULL;
115+
int numFiles = 0;
113116
int mapnum;
117+
int fileno;
118+
119+
old_dir[0] = '\0';
114120

115121
for (mapnum = 0; mapnum < size; mapnum++)
116122
{
117123
char old_file[MAXPGPATH];
118124
char new_file[MAXPGPATH];
119-
struct dirent **namelist = NULL;
120-
int numFiles;
125+
126+
/* Changed tablespaces? Need a new directory scan? */
127+
if (strcmp(maps[mapnum].old_dir, old_dir) != 0)
128+
{
129+
if (numFiles > 0)
130+
{
131+
for (fileno = 0; fileno < numFiles; fileno++)
132+
pg_free(namelist[fileno]);
133+
pg_free(namelist);
134+
}
135+
136+
snprintf(old_dir, sizeof(old_dir), "%s", maps[mapnum].old_dir);
137+
numFiles = pg_scandir(old_dir, &namelist, NULL);
138+
}
121139

122140
/* Copying files might take some time, so give feedback. */
123141

124-
snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_file,
142+
snprintf(old_file, sizeof(old_file), "%s/%u", maps[mapnum].old_dir,
125143
maps[mapnum].old_relfilenode);
126-
snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_file,
144+
snprintf(new_file, sizeof(new_file), "%s/%u", maps[mapnum].new_dir,
127145
maps[mapnum].new_relfilenode);
128146
pg_log(PG_REPORT, OVERWRITE_MESSAGE, old_file);
129147

@@ -139,28 +157,27 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
139157
if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804)
140158
{
141159
/*
142-
* Now copy/link any fsm and vm files, if they exist
160+
* Copy/link any fsm and vm files, if they exist
143161
*/
144162
snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u_",
145163
maps[mapnum].old_relfilenode);
146-
numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
147164

148-
while (numFiles--)
165+
for (fileno = 0; fileno < numFiles; fileno++)
149166
{
150-
snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
151-
namelist[numFiles]->d_name);
152-
snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
153-
maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '_'));
154-
155-
unlink(new_file);
156-
transfer_relfile(pageConverter, old_file, new_file,
157-
maps[mapnum].old_nspname, maps[mapnum].old_relname,
158-
maps[mapnum].new_nspname, maps[mapnum].new_relname);
159-
160-
pg_free(namelist[numFiles]);
167+
if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
168+
strlen(scandir_file_pattern)) == 0)
169+
{
170+
snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
171+
namelist[fileno]->d_name);
172+
snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
173+
maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '_'));
174+
175+
unlink(new_file);
176+
transfer_relfile(pageConverter, old_file, new_file,
177+
maps[mapnum].old_nspname, maps[mapnum].old_relname,
178+
maps[mapnum].new_nspname, maps[mapnum].new_relname);
179+
}
161180
}
162-
163-
pg_free(namelist);
164181
}
165182

166183
/*
@@ -172,23 +189,30 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
172189
*/
173190
snprintf(scandir_file_pattern, sizeof(scandir_file_pattern), "%u.",
174191
maps[mapnum].old_relfilenode);
175-
numFiles = pg_scandir(maps[mapnum].old_file, &namelist, dir_matching_filenames);
176192

177-
while (numFiles--)
193+
for (fileno = 0; fileno < numFiles; fileno++)
178194
{
179-
snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_file,
180-
namelist[numFiles]->d_name);
181-
snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_file,
182-
maps[mapnum].new_relfilenode, strchr(namelist[numFiles]->d_name, '.'));
183-
184-
unlink(new_file);
185-
transfer_relfile(pageConverter, old_file, new_file,
186-
maps[mapnum].old_nspname, maps[mapnum].old_relname,
187-
maps[mapnum].new_nspname, maps[mapnum].new_relname);
195+
if (strncmp(namelist[fileno]->d_name, scandir_file_pattern,
196+
strlen(scandir_file_pattern)) == 0)
197+
{
198+
snprintf(old_file, sizeof(old_file), "%s/%s", maps[mapnum].old_dir,
199+
namelist[fileno]->d_name);
200+
snprintf(new_file, sizeof(new_file), "%s/%u%s", maps[mapnum].new_dir,
201+
maps[mapnum].new_relfilenode, strchr(namelist[fileno]->d_name, '.'));
188202

189-
pg_free(namelist[numFiles]);
203+
unlink(new_file);
204+
transfer_relfile(pageConverter, old_file, new_file,
205+
maps[mapnum].old_nspname, maps[mapnum].old_relname,
206+
maps[mapnum].new_nspname, maps[mapnum].new_relname);
207+
}
190208
}
209+
}
191210

211+
212+
if (numFiles > 0)
213+
{
214+
for (fileno = 0; fileno < numFiles; fileno++)
215+
pg_free(namelist[fileno]);
192216
pg_free(namelist);
193217
}
194218
}

0 commit comments

Comments
 (0)