🌐 AI搜索 & 代理 主页
Skip to content

Commit 3fece18

Browse files
df7cbmsdemlei
df7cb
authored and
msdemlei
committed
GIN index support
For GIN indexing, we degrade all mocs to fixed level MOC_GIN_ORDER (5), and for each pixel, the index stores the set of all mocs where this pixel appears in. The query is degraded to the same level, and the GIN index is queried for these pixels. For intersects (&&) operations, a match is found if at least one pixel matches. Since we are operating on degraded input, rechecking against the original moc is required.
1 parent 0fe578d commit 3fece18

File tree

6 files changed

+365
-1
lines changed

6 files changed

+365
-1
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ crushtest: REGRESS += $(CRUSH_TESTS)
9090
crushtest: installcheck
9191

9292
ifeq ($(has_explain_summary),y)
93-
REGRESS += moc1
93+
REGRESS += moc1 moc100
9494
endif
9595

9696
ifeq ($(pg_version_9_5_plus),y)

‎expected/moc100.out‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
CREATE TABLE moc100 (
2+
ivoid text,
3+
coverage smoc,
4+
ref_system_name text
5+
);
6+
COPY moc100 FROM STDIN;
7+
CREATE INDEX ON moc100 USING GIN (coverage);
8+
SELECT ivoid FROM moc100 WHERE coverage && '4/0' ORDER BY ivoid;
9+
ivoid
10+
------------------------------------------
11+
ivo://byu.arvo/dfbsspec/q/getssa
12+
ivo://cadc.nrc.ca/archive/cfht
13+
ivo://cadc.nrc.ca/archive/hst
14+
ivo://cds.vizier/b/assocdata
15+
ivo://cds.vizier/b/swift
16+
ivo://cds.vizier/i/241
17+
ivo://cds.vizier/iv/12
18+
ivo://cds.vizier/ix/13
19+
ivo://cds.vizier/j/a+a/316/147
20+
ivo://cds.vizier/j/a+as/105/311
21+
ivo://cds.vizier/j/a+as/122/235
22+
ivo://chivo/gaia/q/dr1
23+
ivo://chivo/openngc/q/data
24+
ivo://cxc.harvard.edu/csc
25+
ivo://irsa.ipac/2mass/catalog/psc
26+
ivo://irsa.ipac/2mass/catalog/xsc
27+
ivo://irsa.ipac/2mass/images/asky-ql
28+
ivo://irsa.ipac/cosmos/images
29+
ivo://irsa.ipac/iras/images/issa
30+
ivo://irsa.ipac/mast/scrapbook
31+
ivo://irsa.ipac/spitzer/images/swire
32+
ivo://mssl.ucl.ac.uk/xmmsuss_dsa/xmmsuss
33+
ivo://ned.ipac/sia
34+
ivo://ned.ipac/tap
35+
ivo://svo.cab/cat/gbs
36+
ivo://svo.cab/cat/uves
37+
ivo://svo.cab/cat/xshooter
38+
ivo://vopdc.iap/fss
39+
ivo://vopdc.obspm/imcce/m4ast
40+
ivo://vopdc.obspm/imcce/miriade
41+
ivo://vopdc.obspm/imcce/skybot
42+
ivo://vopdc.obspm/lesia/bestars/besc
43+
ivo://vopdc.obspm/lesia/bestars/bess
44+
ivo://vopdc.obspm/luth/exoplanet
45+
ivo://vopdc.obspm/luth/hess
46+
(35 rows)
47+
48+
EXPLAIN (ANALYZE, BUFFERS, TIMING OFF, SUMMARY OFF)
49+
SELECT * FROM moc100 WHERE coverage && '4/0';
50+
QUERY PLAN
51+
--------------------------------------------------------------------------------
52+
Seq Scan on moc100 (cost=0.00..6.25 rows=1 width=96) (actual rows=35 loops=1)
53+
Filter: (coverage && '4/0'::smoc)
54+
Rows Removed by Filter: 65
55+
Buffers: shared hit=114
56+
(4 rows)
57+
58+
SET enable_seqscan = off;
59+
EXPLAIN (ANALYZE, BUFFERS, TIMING OFF, SUMMARY OFF)
60+
SELECT * FROM moc100 WHERE coverage && '4/0';
61+
QUERY PLAN
62+
------------------------------------------------------------------------------------------------------------
63+
Bitmap Heap Scan on moc100 (cost=36.00..40.01 rows=1 width=96) (actual rows=35 loops=1)
64+
Recheck Cond: (coverage && '4/0'::smoc)
65+
Heap Blocks: exact=5
66+
Buffers: shared hit=85
67+
-> Bitmap Index Scan on moc100_coverage_idx (cost=0.00..36.00 rows=1 width=0) (actual rows=35 loops=1)
68+
Index Cond: (coverage && '4/0'::smoc)
69+
Buffers: shared hit=9
70+
(7 rows)
71+

‎moc.c‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ PG_FUNCTION_INFO_V1(smoc_disc);
3737
PG_FUNCTION_INFO_V1(smoc_scircle);
3838
PG_FUNCTION_INFO_V1(smoc_spoly);
3939

40+
PG_FUNCTION_INFO_V1(smoc_gin_extract_value);
41+
PG_FUNCTION_INFO_V1(smoc_gin_extract_query);
42+
PG_FUNCTION_INFO_V1(smoc_gin_consistent);
43+
4044
int32 smoc_output_type = 0;
4145

4246
#define LAYDEB 0
@@ -1002,3 +1006,112 @@ smoc_spoly(PG_FUNCTION_ARGS)
10021006
create_moc_release_context(moc_in_context, moc_ret, moc_error_out);
10031007
PG_RETURN_POINTER(moc_ret);
10041008
}
1009+
1010+
/* GIN index ***********************************/
1011+
1012+
Datum
1013+
smoc_gin_extract_value(PG_FUNCTION_ARGS)
1014+
{
1015+
Smoc* moc_a = (Smoc *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
1016+
char* moc_a_base = MOC_BASE(moc_a);
1017+
int32 moc_a_end = VARSIZE(moc_a) - VARHDRSZ;
1018+
int32* nkeys = (int32 *) PG_GETARG_POINTER(1);
1019+
int32 nalloc = 4;
1020+
Datum* keys = palloc(nalloc * sizeof(Datum));
1021+
1022+
*nkeys = 0;
1023+
1024+
for (int32 a = moc_a->data_begin; a < moc_a_end; a = next_interval(a))
1025+
{
1026+
moc_interval *x = MOC_INTERVAL(moc_a_base, a);
1027+
1028+
int shift = 2 * (HEALPIX_MAX_ORDER - MOC_GIN_ORDER); // degrade to MOC_GIN_ORDER
1029+
int32 first = (x->first >> shift); // set low bits to zero
1030+
hpint64 low_bits_one = (1L << shift) - 1;
1031+
int32 second = ((x->second + low_bits_one) >> shift); // round low bits up
1032+
Assert(shift > 32); // internal GIN datatype isn't 64 bits
1033+
1034+
// split interval into individual pixels of order MOC_GIN_ORDER
1035+
for (int32 p = first; p < second; p++)
1036+
{
1037+
if (*nkeys > 0 && keys[*nkeys - 1] == p) // has (larger) pixel already been added?
1038+
continue;
1039+
if (*nkeys >= nalloc)
1040+
{
1041+
nalloc *= 2;
1042+
Assert(nalloc < 1000000);
1043+
keys = repalloc(keys, nalloc * sizeof(Datum));
1044+
}
1045+
keys[(*nkeys)++] = Int32GetDatum(p);
1046+
}
1047+
}
1048+
1049+
PG_RETURN_POINTER(keys);
1050+
}
1051+
1052+
Datum
1053+
smoc_gin_extract_query(PG_FUNCTION_ARGS)
1054+
{
1055+
Smoc* moc_a = (Smoc *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
1056+
char* moc_a_base = MOC_BASE(moc_a);
1057+
int32 moc_a_end = VARSIZE(moc_a) - VARHDRSZ;
1058+
int32* nkeys = (int32 *) PG_GETARG_POINTER(1);
1059+
//StrategyNumber st = PG_GETARG_UINT16(2);
1060+
int32 nalloc = 4;
1061+
Datum* keys = palloc(nalloc * sizeof(Datum));
1062+
1063+
*nkeys = 0;
1064+
1065+
//Assert(st == 1);
1066+
1067+
for (int32 a = moc_a->data_begin; a < moc_a_end; a = next_interval(a))
1068+
{
1069+
moc_interval *x = MOC_INTERVAL(moc_a_base, a);
1070+
1071+
int shift = 2 * (HEALPIX_MAX_ORDER - MOC_GIN_ORDER); // degrade to MOC_GIN_ORDER
1072+
int32 first = (x->first >> shift); // set low bits to zero
1073+
hpint64 low_bits_one = (1L << shift) - 1;
1074+
int32 second = ((x->second + low_bits_one) >> shift); // round low bits up
1075+
Assert(shift > 32); // internal GIN datatype isn't 64 bits
1076+
1077+
// split interval into individual pixels of order MOC_GIN_ORDER
1078+
for (int32 p = first; p < second; p++)
1079+
{
1080+
if (*nkeys > 0 && keys[*nkeys - 1] == p) // has (larger) pixel already been added?
1081+
continue;
1082+
if (*nkeys >= nalloc)
1083+
{
1084+
nalloc *= 2;
1085+
Assert(nalloc < 1000000);
1086+
keys = repalloc(keys, nalloc * sizeof(Datum));
1087+
}
1088+
keys[(*nkeys)++] = Int32GetDatum(p);
1089+
}
1090+
}
1091+
1092+
PG_RETURN_POINTER(keys);
1093+
}
1094+
1095+
Datum
1096+
smoc_gin_consistent(PG_FUNCTION_ARGS)
1097+
{
1098+
bool* check = (bool *) PG_GETARG_POINTER(0);
1099+
//StrategyNumber st = PG_GETARG_UINT16(1);
1100+
//Smoc* moc_a = (Smoc *) PG_DETOAST_DATUM(PG_GETARG_DATUM(2));
1101+
//int32 moc_a_end = VARSIZE(moc_a) - VARHDRSZ;
1102+
int32 nkeys = PG_GETARG_INT32(3);
1103+
bool* recheck = (bool *) PG_GETARG_POINTER(5);
1104+
1105+
//Assert(st == 1);
1106+
1107+
for (int i = 0; i < nkeys; i++)
1108+
{
1109+
if (check[i])
1110+
{
1111+
*recheck = true;
1112+
PG_RETURN_BOOL(true);
1113+
}
1114+
}
1115+
1116+
PG_RETURN_BOOL(false);
1117+
}

‎pgs_moc.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,32 @@ Datum smoc_disc(PG_FUNCTION_ARGS);
8989
Datum smoc_scircle(PG_FUNCTION_ARGS);
9090
Datum smoc_spoly(PG_FUNCTION_ARGS);
9191

92+
Datum smoc_gin_extract_value(PG_FUNCTION_ARGS);
93+
Datum smoc_gin_extract_query(PG_FUNCTION_ARGS);
94+
Datum smoc_gin_consistent(PG_FUNCTION_ARGS);
95+
9296
/* parsing subroutines */
9397
hpint64 readNumber(const char*, int*);
9498
char readChar(const char*, int*);
9599

100+
static inline int32
101+
next_interval(int32 a)
102+
{
103+
int32 mod;
104+
105+
a += MOC_INTERVAL_SIZE;
106+
107+
// page bumps
108+
mod = (a + MOC_INTERVAL_SIZE) % PG_TOAST_PAGE_FRAGMENT;
109+
if (mod > 0 && mod < MOC_INTERVAL_SIZE)
110+
a += MOC_INTERVAL_SIZE - mod;
111+
112+
return a;
113+
}
114+
96115
#define MOC_AREA_ALL_SKY 3458764513820540928
97116

117+
#define MOC_GIN_ORDER 5 /* order 5 has 12 * 4^5 = 12288 pixels */
118+
#define MOC_GIN_STRATEGY_INTERSECTS 1
119+
98120
#endif

‎pgs_moc_ops.sql.in‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,3 +280,41 @@ CREATE FUNCTION smoc ("order" int, spoly)
280280
STRICT;
281281

282282
COMMENT ON FUNCTION smoc ("order" int, spoly) IS 'constructs smoc of given order from spoly';
283+
284+
-- GIN index
285+
286+
CREATE FUNCTION smoc_gin_extract_value (smoc, internal, internal)
287+
RETURNS internal
288+
AS 'MODULE_PATHNAME'
289+
LANGUAGE C
290+
PARALLEL SAFE
291+
IMMUTABLE
292+
STRICT;
293+
294+
CREATE FUNCTION smoc_gin_extract_query (smoc, internal, int2, internal, internal, internal, internal)
295+
RETURNS internal
296+
AS 'MODULE_PATHNAME'
297+
LANGUAGE C
298+
PARALLEL SAFE
299+
IMMUTABLE
300+
STRICT;
301+
302+
CREATE FUNCTION smoc_gin_consistent (internal, int2, smoc, int4, internal, internal, internal, internal)
303+
RETURNS boolean
304+
AS 'MODULE_PATHNAME'
305+
LANGUAGE C
306+
PARALLEL SAFE
307+
IMMUTABLE
308+
STRICT;
309+
310+
CREATE OPERATOR CLASS smoc_gin_ops
311+
DEFAULT FOR TYPE smoc USING gin AS
312+
OPERATOR 1 &&,
313+
OPERATOR 2 <@,
314+
FUNCTION 1 btint4cmp (int4, int4),
315+
FUNCTION 2 smoc_gin_extract_value (smoc, internal, internal),
316+
FUNCTION 3 smoc_gin_extract_query (smoc, internal, int2, internal, internal, internal, internal),
317+
FUNCTION 4 smoc_gin_consistent (internal, int2, smoc, int4, internal, internal, internal, internal),
318+
--FUNCTION 5 smoc_gin_compare_partial (),
319+
--FUNCTION 6 smoc_gin_tri_consistent (),
320+
STORAGE int4;

‎sql/moc100.sql‎

Lines changed: 120 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)