Update to Postgres 16 parser, support recent emscripten

.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Set up node environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Set up emscripten
+        uses: mymindstorm/setup-emsdk@v14
+      - name: Build library
+        run: make
+      - name: Runs tests
+        run: NODE_OPTIONS="$NODE_OPTIONS --experimental-vm-modules" npx jest

.gitignore

@@ -1,2 +1,6 @@
 parser/
 tmp/
+node_modules/
+
+pg_query.js
+pg_query_wasm.js

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 5.1.0    2024-09-04
+
+* Update to Postgres 16 parser, match libpg_query versioning scheme
+* Add optional WASM build output in "pg_query_wasm.js" file
+* Adjust function names to use camelCase naming instead of under_score_case [Philip Trauner](https://github.com/PhilipTrauner)
+* Export as ESM module (note this may require use of a transpiler when importing this library) [Philip Trauner](https://github.com/PhilipTrauner)
+* Add support for scan function (pg_query_scan) [Philip Trauner](https://github.com/PhilipTrauner)
+* Support newer emscripten versions [Julz Hoben](https://github.com/neighbaa) [Philip Trauner](https://github.com/PhilipTrauner)
+
+
 ## 0.1.0    2018-09-11
 
 * Update to Postgres 10 parser [Alexey Shamrin](https://github.com/shamrin) [#6](https://github.com/lfittl/pg-query-emscripten/pull/6)

Makefile

@@ -1,50 +1,108 @@
-LIB_PG_QUERY_TAG = 10-1.0.2
-
-root_dir := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-TMPDIR = $(root_dir)/tmp
-LIBDIR = $(TMPDIR)/libpg_query
-LIBDIRGZ = $(TMPDIR)/libpg_query-$(LIB_PG_QUERY_TAG).tar.gz
-
-default:
-	@echo "Run 'make update_source' if you want to regenerate the JS library"
-
-.PHONY: flatten_source fix_pg_config update_source
-
-$(LIBDIR): $(LIBDIRGZ)
-	mkdir -p $(LIBDIR)
-	cd $(TMPDIR); tar -xzf $(LIBDIRGZ) -C $(LIBDIR) --strip-components=1
-
-$(LIBDIRGZ):
-	mkdir -p $(TMPDIR)
-	curl -o $(LIBDIRGZ) https://codeload.github.com/lfittl/libpg_query/tar.gz/$(LIB_PG_QUERY_TAG)
-
-flatten_source: $(LIBDIR)
-	mkdir -p parser
-	rm -f parser/*.{c,h}
-	rm -fr parser/include
-	# Reduce everything down to one directory
-	cp -a $(LIBDIR)/src/* parser/
-	mv parser/postgres/* parser/
-	rmdir parser/postgres
-	cp -a $(LIBDIR)/pg_query.h parser/include
-	# Make sure every .c file in the top-level directory is its own translation unit
-	mv parser/*{_conds,_defs,_helper,_random}.c parser/include
-
-fix_pg_config:
-	echo "#undef HAVE_SIGSETJMP" >> parser/include/pg_config.h
-	echo "#undef HAVE_SPINLOCKS" >> parser/include/pg_config.h
-	echo "#undef PG_INT128_TYPE" >> parser/include/pg_config.h
-
-update_source: flatten_source fix_pg_config
-	emcc -O3 -o pg_query.o -Iparser/include parser/*.c
-	em++ -s EXPORTED_FUNCTIONS="['_normalize', '_parse', '_parse_plpgsql', '_fingerprint']" -Iparser/include -O3 --bind --pre-js module.js --memory-init-file 0 -s "WASM=0" -o tmp/pg_query_raw.js pg_query.o entry.cpp
-	rm -f pg_query.o
-	echo "var PgQuery = (function () {" > pg_query.js
-	cat tmp/pg_query_raw.js >> pg_query.js
-	echo "return { normalize: Module.normalize, parse: Module.parse, parse_plpgsql: Module.parse_plpgsql, fingerprint: Module.fingerprint };" >> pg_query.js
-	echo "})();" >> pg_query.js
-	echo "if (typeof module !== 'undefined') module.exports = PgQuery;" >> pg_query.js
-	echo "if (typeof define === 'function') define(PgQuery);" >> pg_query.js
+LIB_PG_QUERY_TAG := 16-5.1.0
+
+BUILD_DIR := tmp/$(LIB_PG_QUERY_TAG)
+FLATTENED_LIB_DIR := $(BUILD_DIR)/flattened
+OBJECT_DIR := $(BUILD_DIR)/object
+LIB_DIR := $(BUILD_DIR)/libpg_query
+LIB_ARCHIVE := $(BUILD_DIR)/libpg_query.tar.gz
+
+WASM ?= 0
+
+ifeq ($(WASM),1)
+ARTIFACT := pg_query_wasm.js
+else
+ifeq ($(WASM),0)
+ARTIFACT := pg_query.js
+else
+$(error Invalid $$WASM value "$(WASM)" (can either by `0` or `1`))
+endif
+endif
+
+.PHONY: flatten test
+.NOTPARALLEL: flatten
+
+all: flatten
+
+# Generates the source tree that `pg_query.js` operates on
+flatten:
+	mkdir -p "$(BUILD_DIR)"
+
+	if [ ! -e "$(LIB_ARCHIVE)" ]; then \
+		curl -o "$(LIB_ARCHIVE)" \
+			"https://codeload.github.com/pganalyze/libpg_query/tar.gz/$(LIB_PG_QUERY_TAG)"; \
+	fi
+
+	rm -rf "$(LIB_DIR)"
+	mkdir "$(LIB_DIR)"
+
+	tar -xf "$(LIB_ARCHIVE)" -C "$(LIB_DIR)" --strip-components 1
+
+	rm -rf "$(FLATTENED_LIB_DIR)"
+	mkdir -p "$(FLATTENED_LIB_DIR)"
+
+	cp -a $(LIB_DIR)/src/* "$(FLATTENED_LIB_DIR)"
+	mv $(FLATTENED_LIB_DIR)/postgres/include "$(FLATTENED_LIB_DIR)/include/postgres"
+	mv $(FLATTENED_LIB_DIR)/postgres/* "$(FLATTENED_LIB_DIR)"
+	rmdir "$(FLATTENED_LIB_DIR)/postgres"
+	cp -a "$(LIB_DIR)/pg_query.h" "$(FLATTENED_LIB_DIR)/include"
+
+	# Protobuf definitions
+	# TODO: This generated .ts file seems to run the typescript compiler out of memory, possible due to recursive references? (see https://github.com/microsoft/TypeScript/issues/53087)
+	#npm install ts-proto
+	#protoc --proto_path=$(LIB_DIR)/protobuf --plugin=./node_modules/.bin/protoc-gen-ts_proto --ts_proto_out=. $(LIB_DIR)/protobuf/pg_query.proto
+	mkdir -p $(FLATTENED_LIB_DIR)//include/protobuf
+	cp -a $(LIB_DIR)/protobuf/*.h $(FLATTENED_LIB_DIR)/include/protobuf
+	cp -a $(LIB_DIR)/protobuf/*.c $(FLATTENED_LIB_DIR)/
+	# Protobuf library code
+	mkdir -p $(FLATTENED_LIB_DIR)//include/protobuf-c
+	cp -a $(LIB_DIR)/vendor/protobuf-c/*.h $(FLATTENED_LIB_DIR)/include
+	cp -a $(LIB_DIR)/vendor/protobuf-c/*.h $(FLATTENED_LIB_DIR)/include/protobuf-c
+	cp -a $(LIB_DIR)/vendor/protobuf-c/*.c $(FLATTENED_LIB_DIR)/
+	# xxhash library code
+	mkdir -p $(FLATTENED_LIB_DIR)//include/xxhash
+	cp -a $(LIB_DIR)/vendor/xxhash/*.h $(FLATTENED_LIB_DIR)/include
+	cp -a $(LIB_DIR)/vendor/xxhash/*.h $(FLATTENED_LIB_DIR)/include/xxhash
+	cp -a $(LIB_DIR)/vendor/xxhash/*.c $(FLATTENED_LIB_DIR)/
+
+	echo "#undef HAVE_SIGSETJMP" >> "$(FLATTENED_LIB_DIR)/include/pg_config.h"
+	echo "#undef HAVE_SPINLOCKS" >> "$(FLATTENED_LIB_DIR)/include/pg_config.h"
+	echo "#undef PG_INT128_TYPE" >> "$(FLATTENED_LIB_DIR)/include/pg_config.h"
+
+	$(MAKE) $(ARTIFACT)
+
+SOURCES := $(wildcard $(FLATTENED_LIB_DIR)/*.c)
+OBJECTS := $(patsubst $(FLATTENED_LIB_DIR)/%.c,$(OBJECT_DIR)/%.o, $(SOURCES))
+
+$(OBJECT_DIR)/%.o: $(FLATTENED_LIB_DIR)/%.c
+	@mkdir -p $(@D)
+	emcc \
+		-I $(FLATTENED_LIB_DIR)/include \
+		-I $(FLATTENED_LIB_DIR)/include/postgres \
+		-O3 -c $< -o $@
+
+
+$(ARTIFACT): $(OBJECTS) entry.cpp module.js
+	em++ \
+		-I $(FLATTENED_LIB_DIR)/include \
+		-I $(FLATTENED_LIB_DIR)/include/postgres \
+		-s ALLOW_MEMORY_GROWTH=1 \
+		-s ASSERTIONS=0 \
+		-s EXPORTED_RUNTIME_METHODS="['ALLOC_STACK','allocate']" \
+		-s EXPORTED_FUNCTIONS="['_free']" \
+		-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=allocate \
+		-s ENVIRONMENT=web \
+		-s SINGLE_FILE=1 \
+		-s MODULARIZE=1 \
+		-s EXPORT_NAME="pgQuery" \
+		-s WASM=$(WASM) \
+		-s USE_ES6_IMPORT_META=0 \
+		-s EXPORT_ES6=1 \
+		-s ERROR_ON_UNDEFINED_SYMBOLS=0 \
+		-o $(ARTIFACT) --bind -O3 --no-entry --pre-js module.js \
+		$(OBJECTS) entry.cpp
 
 clean:
-	-@ $(RM) -r $(TMPDIR)
+	-@ $(RM) -r $(BUILD_DIR)
+
+test:
+	NODE_OPTIONS=--experimental-vm-modules yarn test

README.md

@@ -1,37 +1,33 @@
-## pg-query-emscripten [ ![](https://img.shields.io/npm/v/pg-query-emscripten.svg)](https://www.npmjs.com/package/pg-query-emscripten)
+## pg-query-emscripten [![](https://img.shields.io/npm/v/pg-query-emscripten.svg)](https://www.npmjs.com/package/pg-query-emscripten)
 
 Parse any valid PostgreSQL query in your browser using Javascript!
 
 This builds a pure Javascript port of [libpg_query](https://github.com/lfittl/libpg_query) using emscripten, that allows you to parse SQL in the browser into the PostgreSQL parse tree.
 
 Example use cases might include automatically checking for bad query patterns (e.g. LIMIT/OFFSET), understanding which tables a query references, or using structural pg_dump output to produce a schema diagram on the fly.
 
-### Usage (Plain JS)
-
-```html
-<script src="https://unpkg.com/pg-query-emscripten"></script>
-<script>
-  console.log(PgQuery.parse("SELECT 1"));
-</script>
-```
-
-### Usage (npm)
+### Usage
 
 ```
 npm install pg-query-emscripten --save
 ```
 
-Then import using your favorite tool, e.g.
-
 ```javascript
-import PgQuery from 'pg-query-emscripten';
+import Module from "pg-query-emscripten";
+
+let pgQuery;
+
+(async () => {
+  pgQuery = await new Module();
 
-console.log(PgQuery.parse("SELECT 1"));
+  console.log(pgQuery.parse("select 1"));
+})();
 ```
 
 ### Author
 
-* [Lukas Fittl](https://github.com/lfittl)
+- [Lukas Fittl](https://github.com/lfittl)
+- [Philip Trauner](https://github.com/PhilipTrauner)
 
 ### LICENSE