🌐 AI搜索 & 代理 主页
Skip to content

Commit 45c0024

Browse files
tglsfdctglsfdc
committed
Remove fixed-size literal buffer from ecpg's lexer (same
fix recently applied to backend's lexer). I see that YY_USES_REJECT still gets defined for this lexer, which means it's going to have trouble parsing really long tokens. Not sure if it's worth doing anything about that or not; I don't have the interest right now to understand why ecpg's additions to the syntax cause this problem...
1 parent abceb20 commit 45c0024

File tree

2 files changed

+107
-84
lines changed

2 files changed

+107
-84
lines changed

‎src/interfaces/ecpg/preproc/Makefile‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,19 @@ OBJ=preproc.o pgc.o type.o ecpg.o ecpg_keywords.o \
1515

1616
all:: ecpg
1717

18+
# Rule that really do something.
19+
ecpg: $(OBJ)
20+
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS)
21+
1822
preproc.c preproc.h: preproc.y
1923
$(YACC) $(YFLAGS) $<
2024
mv y.tab.c preproc.c
2125
mv y.tab.h preproc.h
2226

27+
pgc.c: pgc.l
28+
$(LEX) $<
29+
mv lex.yy.c pgc.c
30+
2331
clean:
2432
rm -f *.o core a.out ecpg$(X) *~ *.output
2533
# And the garbage that might have been left behind by partial build:
@@ -33,19 +41,9 @@ install: all
3341
uninstall:
3442
rm -f $(BINDIR)/ecpg
3543

36-
# Rule that really do something.
37-
ecpg: $(OBJ)
38-
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS)
39-
40-
pgc.c: pgc.l
41-
$(LEX) $<
42-
sed -e 's/#define YY_BUF_SIZE .*/#define YY_BUF_SIZE 65536/' \
43-
<lex.yy.c >pgc.c
44-
rm -f lex.yy.c
45-
46-
preproc.o : preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
47-
type.o : ../include/ecpgtype.h
48-
pgc.o : ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
44+
preproc.o: preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
45+
type.o: ../include/ecpgtype.h
46+
pgc.o: ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
4947
keywords.o: ../include/ecpgtype.h preproc.h
5048
c_keywords.o: ../include/ecpgtype.h preproc.h
5149
ecpg_keywords.o: ../include/ecpgtype.h preproc.h

‎src/interfaces/ecpg/preproc/pgc.l‎

Lines changed: 96 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,61 @@
1-
2-
/* This is a modified version of src/backend/parser/scan.l */
31
%{
2+
/*-------------------------------------------------------------------------
3+
*
4+
* pgc.l
5+
* lexical scanner for ecpg
6+
*
7+
* This is a modified version of src/backend/parser/scan.l
8+
*
9+
*
10+
* Copyright (c) 1994, Regents of the University of California
11+
*
12+
*
13+
* IDENTIFICATION
14+
* $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/pgc.l,v 1.45 1999/10/22 23:14:50 tgl Exp $
15+
*
16+
*-------------------------------------------------------------------------
17+
*/
418
#include <ctype.h>
519
#include <sys/types.h>
620
#include <limits.h>
721
#include <errno.h>
822

923
#include "postgres.h"
24+
1025
#ifndef PATH_MAX
1126
#include <sys/param.h>
1227
#define PATH_MAX MAXPATHLEN
1328
#endif
29+
1430
#include "miscadmin.h"
15-
#include "nodes/pg_list.h"
1631
#include "nodes/parsenodes.h"
32+
#include "nodes/pg_list.h"
1733
#include "parser/gramparse.h"
1834
#include "parser/scansup.h"
1935
#include "extern.h"
2036
#include "preproc.h"
2137
#include "utils/builtins.h"
2238

23-
#ifdef YY_READ_BUF_SIZE
24-
#undef YY_READ_BUF_SIZE
25-
#endif
26-
#define YY_READ_BUF_SIZE MAX_PARSE_BUFFER
27-
2839
/* some versions of lex define this as a macro */
2940
#if defined(yywrap)
3041
#undef yywrap
3142
#endif /* yywrap */
3243

3344
extern YYSTYPE yylval;
34-
int llen;
35-
char literal[MAX_PARSE_BUFFER];
45+
46+
/*
47+
* literalbuf is used to accumulate literal values when multiple rules
48+
* are needed to parse a single literal. Call startlit to reset buffer
49+
* to empty, addlit to add text. Note that the buffer is permanently
50+
* malloc'd to the largest size needed so far in the current run.
51+
*/
52+
static char *literalbuf = NULL; /* expandable buffer */
53+
static int literallen; /* actual current length */
54+
static int literalalloc; /* current allocated buffer size */
55+
56+
#define startlit() (literalbuf[0] = '\0', literallen = 0)
57+
static void addlit(char *ytext, int yleng);
58+
3659
int before_comment;
3760

3861
struct _yy_buffer { YY_BUFFER_STATE buffer;
@@ -142,16 +165,14 @@ self [,()\[\].;$\:\+\-\*\/\%\^\<\>\=\|]
142165
op_and_self [\~\!\@\#\^\&\|\`\?\$\:\+\-\*\/\%\<\>\=]
143166
operator {op_and_self}+
144167

145-
/* we do not allow unary minus in numbers.
146-
* instead we pass it verbatim to parser. there it gets
147-
* coerced via doNegate() -- Leon aug 20 1999
168+
/* we no longer allow unary minus in numbers.
169+
* instead we pass it separately to parser. there it gets
170+
* coerced via doNegate() -- Leon aug 20 1999
148171
*/
172+
149173
integer {digit}+
150174
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
151-
real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+))
152-
/*
153-
real (((({digit}*\.{digit}+)|({digit}+\.{digit}*))([Ee][-+]?{digit}+)?)|({digit}+[Ee][-+]?{digit}+))
154-
*/
175+
real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+))
155176

156177
param \${integer}
157178

@@ -200,99 +221,82 @@ cppline {space}*#.*(\\{space}*\n)*\n*
200221

201222
<SQL>{xbstart} {
202223
BEGIN(xb);
203-
llen = 0;
204-
*literal = '\0';
224+
startlit();
205225
}
206226
<xb>{xbstop} {
207227
char* endptr;
208228

209229
BEGIN(SQL);
210230
errno = 0;
211-
yylval.ival = strtol((char *)literal,&endptr,2);
231+
yylval.ival = strtol(literalbuf, &endptr, 2);
212232
if (*endptr != '\0' || errno == ERANGE)
213233
yyerror("ERROR: Bad binary integer input!");
214234
return ICONST;
215235
}
216236
<xh>{xhinside} |
217237
<xb>{xbinside} {
218-
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1))
219-
yyerror("ERROR: quoted string parse buffer exceeded");
220-
memcpy(literal+llen, yytext, yyleng+1);
221-
llen += yyleng;
238+
addlit(yytext, yyleng);
222239
}
223240
<xh>{xhcat} |
224241
<xb>{xbcat} {
225242
}
226243

227244
<SQL>{xhstart} {
228245
BEGIN(xh);
229-
llen = 0;
230-
*literal = '\0';
246+
startlit();
231247
}
232248
<xh>{xhstop} {
233249
char* endptr;
234250

235251
BEGIN(SQL);
236252
errno = 0;
237-
yylval.ival = strtol((char *)literal,&endptr,16);
253+
yylval.ival = strtol(literalbuf, &endptr, 16);
238254
if (*endptr != '\0' || errno == ERANGE)
239255
yyerror("ERROR: Bad hexadecimal integer input");
240256
return ICONST;
241257
}
242258

243259
<SQL>{xqstart} {
244260
BEGIN(xq);
245-
llen = 0;
246-
*literal = '\0';
261+
startlit();
247262
}
248263
<xq>{xqstop} {
249264
BEGIN(SQL);
250-
yylval.str = mm_strdup(literal);
265+
yylval.str = mm_strdup(literalbuf);
251266
return SCONST;
252267
}
253268
<xq>{xqdouble} |
254-
<xq>{xqinside} |
269+
<xq>{xqinside} |
255270
<xq>{xqliteral} {
256-
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1))
257-
yyerror("ERROR: quoted string parse buffer exceeded");
258-
memcpy(literal+llen, yytext, yyleng+1);
259-
llen += yyleng;
271+
addlit(yytext, yyleng);
260272
}
261273
<xq>{xqcat} {
262274
}
263275

264276

265277
<SQL>{xdstart} {
266278
BEGIN(xd);
267-
llen = 0;
268-
*literal = '\0';
279+
startlit();
269280
}
270281
<xd>{xdstop} {
271282
BEGIN(SQL);
272-
yylval.str = mm_strdup(literal);
283+
yylval.str = mm_strdup(literalbuf);
273284
return CSTRING;
274285
}
275286
<xd>{xdinside} {
276-
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1))
277-
yyerror("ERROR: quoted string parse buffer exceeded");
278-
memcpy(literal+llen, yytext, yyleng+1);
279-
llen += yyleng;
287+
addlit(yytext, yyleng);
280288
}
281289
{xdstart} {
282290
BEGIN(xdc);
283-
llen = 0;
284-
*literal = '\0';
291+
startlit();
285292
}
286293
<xdc>{xdstop} {
287294
BEGIN(C);
288-
yylval.str = mm_strdup(literal);
295+
yylval.str = mm_strdup(literalbuf);
289296
return CSTRING;
290297
}
291298
<xdc>{xdcinside} {
292-
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1))
293-
yyerror("ERROR: quoted string parse buffer exceeded");
294-
memcpy(literal+llen, yytext, yyleng+1);
295-
llen += yyleng;
299+
addlit(yytext, yyleng);
296300
}
297301
<SQL>{typecast} { return TYPECAST; }
298302
<SQL>{self} { /*
@@ -324,24 +328,24 @@ cppline {space}*#.*(\\{space}*\n)*\n*
324328
{
325329
errno = 0;
326330
yylval.str = mm_strdup((char*)yytext);
327-
return SCONST;
331+
return SCONST;
328332
}
329333
return ICONST;
330334
}
331-
{decimal} {
332-
char* endptr;
335+
{decimal} {
336+
char* endptr;
333337

334-
if (strlen((char *)yytext) <= 17)
335-
{
336-
errno = 0;
337-
yylval.dval = strtod((char *)yytext,&endptr);
338+
if (strlen((char *)yytext) <= 17)
339+
{
340+
errno = 0;
341+
yylval.dval = strtod((char *)yytext,&endptr);
338342
if (*endptr != '\0' || errno == ERANGE)
339343
yyerror("ERROR: Bad float8 input");
340344
return FCONST;
341-
}
342-
yylval.str = mm_strdup((char*)yytext);
343-
return SCONST;
344-
}
345+
}
346+
yylval.str = mm_strdup((char*)yytext);
347+
return SCONST;
348+
}
345349
<C,SQL>{real} {
346350
char* endptr;
347351

@@ -420,7 +424,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
420424
{
421425
errno = 0;
422426
yylval.str = mm_strdup((char*)yytext);
423-
return SCONST;
427+
return SCONST;
424428
}
425429
return ICONST;
426430
}
@@ -486,8 +490,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
486490
<def_ident>{identifier} {
487491
old = mm_strdup(yytext);
488492
BEGIN(def);
489-
llen = 0;
490-
*literal = '\0';
493+
startlit();
491494
}
492495
<def>{space} /* eat the whitespace */
493496
<def>";" {
@@ -498,8 +501,8 @@ cppline {space}*#.*(\\{space}*\n)*\n*
498501
if (strcmp(old, ptr->old) == 0)
499502
{
500503
free(ptr->new);
501-
/* ptr->new = mm_strdup(scanstr(literal));*/
502-
ptr->new = mm_strdup(literal);
504+
/* ptr->new = mm_strdup(scanstr(literalbuf));*/
505+
ptr->new = mm_strdup(literalbuf);
503506
}
504507
}
505508
if (ptr == NULL)
@@ -508,19 +511,16 @@ cppline {space}*#.*(\\{space}*\n)*\n*
508511

509512
/* initial definition */
510513
this->old = old;
511-
/* this->new = mm_strdup(scanstr(literal));*/
512-
this->new = mm_strdup(literal);
514+
/* this->new = mm_strdup(scanstr(literalbuf));*/
515+
this->new = mm_strdup(literalbuf);
513516
this->next = defines;
514517
defines = this;
515518
}
516519

517520
BEGIN(C);
518521
}
519522
<def>[^";"] {
520-
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1))
521-
yyerror("ERROR: define statement parse buffer exceeded");
522-
memcpy(literal+llen, yytext, yyleng+1);
523-
llen += yyleng;
523+
addlit(yytext, yyleng);
524524
}
525525
<C>{exec}{space}{sql}{space}{include} { BEGIN(incl); }
526526
<incl>{space} /* eat the whitespace */
@@ -602,9 +602,34 @@ void
602602
lex_init(void)
603603
{
604604
braces_open = 0;
605+
606+
/* initialize literal buffer to a reasonable but expansible size */
607+
if (literalbuf == NULL)
608+
{
609+
literalalloc = 128;
610+
literalbuf = (char *) malloc(literalalloc);
611+
}
612+
startlit();
613+
605614
BEGIN C;
606615
}
607616

617+
static void
618+
addlit(char *ytext, int yleng)
619+
{
620+
/* enlarge buffer if needed */
621+
if ((literallen+yleng) >= literalalloc)
622+
{
623+
do {
624+
literalalloc *= 2;
625+
} while ((literallen+yleng) >= literalalloc);
626+
literalbuf = (char *) realloc(literalbuf, literalalloc);
627+
}
628+
/* append data --- note we assume ytext is null-terminated */
629+
memcpy(literalbuf+literallen, ytext, yleng+1);
630+
literallen += yleng;
631+
}
632+
608633
int yywrap(void)
609634
{
610635
return 1;

0 commit comments

Comments
 (0)