diff --git a/projects/postgresql/Dockerfile b/projects/postgresql/Dockerfile index c0e000368..ca0db2740 100644 --- a/projects/postgresql/Dockerfile +++ b/projects/postgresql/Dockerfile @@ -19,7 +19,10 @@ FROM gcr.io/oss-fuzz-base/base-builder RUN apt-get update && apt-get install -y make libreadline-dev zlib1g-dev bison flex RUN git clone git://git.postgresql.org/git/postgresql.git +RUN zip postgresql_fuzzer_seed_corpus.zip postgresql/src/test/regress/sql/* WORKDIR postgresql +RUN mkdir bld + COPY fuzzer $SRC/fuzzer -COPY add_fuzzers.diff build.sh $SRC/ +COPY build.sh $SRC/ diff --git a/projects/postgresql/add_fuzzers.diff b/projects/postgresql/add_fuzzers.diff deleted file mode 100644 index 8d7a27470..000000000 --- a/projects/postgresql/add_fuzzers.diff +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/src/backend/Makefile b/src/backend/Makefile -index 9706a95848..4312c346a4 100644 ---- a/src/backend/Makefile -+++ b/src/backend/Makefile -@@ -43,6 +43,8 @@ OBJS = \ - $(top_builddir)/src/common/libpgcommon_srv.a \ - $(top_builddir)/src/port/libpgport_srv.a - -+OBJS_WITHOUT_MAIN = $(filter-out main/objfiles.txt, $(OBJS)) -+ - # We put libpgport and libpgcommon into OBJS, so remove it from LIBS; also add - # libldap and ICU - LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) -@@ -56,7 +58,10 @@ endif - - ########################################################################## - --all: submake-libpgport submake-catalog-headers submake-utils-headers postgres $(POSTGRES_IMP) -+all: submake-libpgport submake-catalog-headers submake-utils-headers postgres fuzzer $(POSTGRES_IMP) -+ -+fuzzer: fuzzer/simple_query_fuzzer \ -+ fuzzer/json_parser_fuzzer - - ifneq ($(PORTNAME), cygwin) - ifneq ($(PORTNAME), win32) -@@ -65,6 +70,12 @@ ifneq ($(PORTNAME), aix) - postgres: $(OBJS) - $(CC) $(CFLAGS) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(LIBS) -o $@ - -+fuzzer/json_parser_fuzzer: fuzzer/json_parser_fuzzer.o $(OBJS_WITHOUT_MAIN) -+ $(CXX) $(CFLAGS) $(call expand_subsys,$^) -o $@ $(LIB_FUZZING_ENGINE) -+ -+fuzzer/simple_query_fuzzer: fuzzer/simple_query_fuzzer.o $(OBJS_WITHOUT_MAIN) -+ $(CXX) $(CFLAGS) $(call expand_subsys,$^) -o $@ $(LIB_FUZZING_ENGINE) -+ - endif - endif - endif diff --git a/projects/postgresql/build.sh b/projects/postgresql/build.sh index 9dec17820..cdd0da757 100644 --- a/projects/postgresql/build.sh +++ b/projects/postgresql/build.sh @@ -14,12 +14,24 @@ # limitations under the License. # ################################################################################ -git apply ../add_fuzzers.diff cp -r $SRC/fuzzer src/backend/ -mkdir bld + +useradd fuzzuser +chown -R fuzzuser . cd bld -../configure --prefix /usr/local/ -make -j$(nproc) +CC="" CXX="" CFLAGS="" CXXFLAGS="" su fuzzuser -c ../configure +cd src/backend/fuzzer +su fuzzuser -c "make createdb" +chown -R root . +cp -r temp $OUT/ +cd ../../.. +cp -r tmp_install $OUT/ +make clean -cp src/backend/fuzzer/*_fuzzer $OUT/ +../configure +make +cd src/backend/fuzzer +make fuzzer +cp *_fuzzer $OUT/ +cp $SRC/postgresql_fuzzer_seed_corpus.zip $OUT/ diff --git a/projects/postgresql/fuzzer/Makefile b/projects/postgresql/fuzzer/Makefile index ee40efadc..6683794b8 100644 --- a/projects/postgresql/fuzzer/Makefile +++ b/projects/postgresql/fuzzer/Makefile @@ -5,14 +5,60 @@ # src/backend/fuzzer/Makefile # #------------------------------------------------------------------------- - subdir = src/backend/fuzzer top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +subsysfilename = objfiles.txt + +SUBDIROBJS = $(SUBDIRS:%=%/$(subsysfilename)) + +# make function to expand objfiles.txt contents +expand_subsys = $(foreach file,$(1),$(if $(filter %/objfiles.txt,$(file)),$(patsubst ../../src/backend/%,%,$(addprefix $(top_builddir)/,$(shell cat $(file)))),$(file))) + +objfiles.txt: Makefile $(SUBDIROBJS) $(OBJS_FUZZERS) +# Don't rebuild the list if only the OBJS have changed. + $(if $(filter-out $(OBJS_FUZZERS),$?),( $(if $(SUBDIROBJS),cat $(SUBDIROBJS); )echo $(addprefix $(subdir)/,$(OBJS_FUZZERS)) ) >$@,touch $@) + +SUBDIRS = ../access ../bootstrap ../catalog ../parser ../commands ../executor ../foreign ../lib ../libpq \ + ../main ../nodes ../optimizer ../partitioning ../port ../postmaster \ + ../regex ../replication ../rewrite \ + ../statistics ../storage ../tcop ../tsearch ../utils $(top_builddir)/src/timezone \ + ../jit + OBJS = \ - json_parser_fuzzer.o \ - simple_query_fuzzer.o + $(LOCALOBJS) \ + $(SUBDIROBJS) \ + $(top_builddir)/src/common/libpgcommon_srv.a \ + $(top_builddir)/src/port/libpgport_srv.a \ -include $(top_srcdir)/src/backend/common.mk +OBJS_FUZZERS = $(filter-out ../main/objfiles.txt, $(OBJS)) +createdb: dbfuzz + +fuzzer: simple_query_fuzzer \ + json_parser_fuzzer + + +json_parser_fuzzer: json_parser_fuzzer.o $(OBJS_FUZZERS) + $(CXX) $(CFLAGS) $(call expand_subsys,$^) -o $@ $(LIB_FUZZING_ENGINE) + +simple_query_fuzzer: simple_query_fuzzer.o $(OBJS_FUZZERS) + $(CXX) $(CFLAGS) $(call expand_subsys,$^) -o $@ $(LIB_FUZZING_ENGINE) + +json_parser_fuzzer.o: json_parser_fuzzer.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $^ + +simple_query_fuzzer.o: simple_query_fuzzer.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $^ + + +dbfuzz: dbfuzz.o | submake-libpgport temp-install + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@ \ + && PATH="$(abs_top_builddir)/tmp_install$(bindir):$$PATH" LD_LIBRARY_PATH="$(abs_top_builddir)/tmp_install/usr/local/pgsql/lib" ./dbfuzz + +dbfuzz.o: dbfuzz.c $(top_builddir)/src/port/pg_config_paths.h +dbfuzz.o: override CPPFLAGS := $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_builddir)/../src/test/regress '-DSHELLPROG="$(SHELL)"' + +$(top_builddir)/src/port/pg_config_paths.h: | submake-libpgport + $(MAKE) -C $(top_builddir)/src/port pg_config_paths.h diff --git a/projects/postgresql/fuzzer/dbfuzz.c b/projects/postgresql/fuzzer/dbfuzz.c new file mode 100644 index 000000000..9a9bf79a7 --- /dev/null +++ b/projects/postgresql/fuzzer/dbfuzz.c @@ -0,0 +1,146 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +/////////////////////////////////////////////////////////////////////////// + +/*------------------------------------------------------------------------- + * + * + * This code is released under the terms of the PostgreSQL License. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include +#include + +#include "common/logging.h" +#include "common/restricted_token.h" +#include "libpq/pqcomm.h" +#include "pg_config_paths.h" +#include "pg_regress.h" + +const char *progname = "progname"; +static char *shellprog = SHELLPROG; +char *outputdir = "."; +static char *temp_instance = NULL; +static int port = -1; +static const char *sockdir; +static PID_TYPE postmaster_pid = INVALID_PID; + +static void psql_command(const char *database, const char *query, ...) { + char query_formatted[1024]; + char query_escaped[2048]; + char psql_cmd[MAXPGPATH + 2048]; + va_list args; + char *s; + char *d; + + va_start(args, query); + vsnprintf(query_formatted, sizeof(query_formatted), query, args); + va_end(args); + + d = query_escaped; + for (s = query_formatted; *s; s++) { + if (strchr("\\\"$`", *s)) + *d++ = '\\'; + *d++ = *s; + } + *d = '\0'; + + snprintf(psql_cmd, sizeof(psql_cmd), "\"%s%spsql\" -X -c \"%s\" \"%s\"", + "", "", query_escaped, database); + + system(psql_cmd); +} + +PID_TYPE +spawn_process(const char *cmdline) { + pid_t pid; + pid = fork(); + if (pid == 0) { + char *cmdline2; + cmdline2 = psprintf("exec %s", cmdline); + execl(shellprog, shellprog, "-c", cmdline2, (char *)NULL); + fprintf(stderr, _("%s: could not exec \"%s\": %s\n"), progname, shellprog, + strerror(errno)); + _exit(1); + } + + return pid; +} + +int main() { + int i; + char buf[MAXPGPATH * 4]; + char buf2[MAXPGPATH * 4]; + char *db_name = "./dbfuzz"; + int wait_seconds = 60; + + pg_logging_init(db_name); + progname = get_progname(db_name); + set_pglocale_pgservice(db_name, PG_TEXTDOMAIN("pg_dbfuzz")); + get_restricted_token(); + + temp_instance = make_absolute_path("./temp"); + port = 0xC000 | (PG_VERSION_NUM & 0x3FFF); + outputdir = make_absolute_path(outputdir); + sockdir = mkdtemp(psprintf("/tmp/pg_dbfuzz-XXXXXX")); + putenv(psprintf("PGHOST=%s", sockdir)); + + mkdir(temp_instance, S_IRWXU | S_IRWXG | S_IRWXO); + + snprintf(buf, sizeof(buf), "%s/log", outputdir); + mkdir(buf, S_IRWXU | S_IRWXG | S_IRWXO); + + snprintf(buf, sizeof(buf), + "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync > " + "\"%s/log/initdb.log\" 2>&1", + "", "", temp_instance, outputdir); + system(buf); + + snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance); + + snprintf(buf2, sizeof(buf2), "\"%s%spsql\" -X postgres <%s 2>%s", "", "", + DEVNULL, DEVNULL); + + snprintf(buf, sizeof(buf), + "\"%s%spostgres\" -D \"%s/data\" -F%s " + "-c \"listen_addresses=%s\" -k \"%s\" " + "> \"%s/log/postmaster.log\" 2>&1", + "", "", temp_instance, "", "", sockdir, outputdir); + + postmaster_pid = spawn_process(buf); + + for (i = 0; i < wait_seconds; i++) { + if (system(buf2) == 0) + break; + waitpid(postmaster_pid, NULL, WNOHANG); + pg_usleep(1000000L); + } + + psql_command("postgres", "CREATE DATABASE \"%s\" TEMPLATE=template0%s", + "dbfuzz", ""); + + snprintf(buf, sizeof(buf), "\"%s%spg_ctl\" stop -D \"%s/data\" -s", "", + "", temp_instance); + system(buf); + + rmdir(sockdir); + return 0; +} diff --git a/projects/postgresql/fuzzer/json_parser_fuzzer.c b/projects/postgresql/fuzzer/json_parser_fuzzer.c index 244e46910..24c9ec377 100644 --- a/projects/postgresql/fuzzer/json_parser_fuzzer.c +++ b/projects/postgresql/fuzzer/json_parser_fuzzer.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "common/jsonapi.h" #include "mb/pg_wchar.h" +#include "miscadmin.h" #include "utils/memutils.h" #include "utils/memdebug.h" @@ -25,12 +26,19 @@ const char *progname = "progname"; ** fuzzed input. */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - MemoryContextInit(); sigjmp_buf local_sigjmp_buf; - char *buffer = (char *) calloc(size+1, sizeof(char)); + char *buffer; + JsonSemAction sem; + JsonLexContext *lex; + + buffer = (char *) calloc(size+1, sizeof(char)); memcpy(buffer, data, size); - JsonSemAction sem = nullSemAction; - JsonLexContext *lex = makeJsonLexContextCstringLen(buffer, size+1, PG_UTF8, true); + + MemoryContextInit(); + set_stack_base(); + sem = nullSemAction; + lex = makeJsonLexContextCstringLen(buffer, size+1, PG_UTF8, true); + if(!sigsetjmp(local_sigjmp_buf,0)){ error_context_stack = NULL; PG_exception_stack = &local_sigjmp_buf; diff --git a/projects/postgresql/fuzzer/simple_query_fuzzer.c b/projects/postgresql/fuzzer/simple_query_fuzzer.c index 64f35501c..f80b30562 100644 --- a/projects/postgresql/fuzzer/simple_query_fuzzer.c +++ b/projects/postgresql/fuzzer/simple_query_fuzzer.c @@ -13,46 +13,151 @@ // limitations under the License. #include "postgres.h" -#include "parser/gramparse.h" -#include "parser/parser.h" -#include "parser/analyze.h" -#include "utils/memutils.h" -#include "utils/memdebug.h" -#include "rewrite/rewriteHandler.h" + +#include "access/xlog.h" +#include "access/xact.h" +#include "common/username.h" +#include "executor/spi.h" +#include "jit/jit.h" +#include "libpq/libpq.h" +#include "libpq/pqsignal.h" +#include "miscadmin.h" #include "optimizer/optimizer.h" +#include "parser/analyze.h" +#include "parser/parser.h" +#include "storage/proc.h" +#include "tcop/tcopprot.h" +#include "utils/datetime.h" +#include "utils/memutils.h" +#include "utils/portal.h" #include "utils/snapmgr.h" -#include "nodes/params.h" -#include "nodes/plannodes.h" -#include "nodes/pg_list.h" +#include "utils/timeout.h" -const char *progname = "progname"; +const char *progname; +static const char *userDoption; +static MemoryContext row_description_context = NULL; +static StringInfoData row_description_buf; +static const char *dbname = NULL; +static const char *username = NULL; -List *plan_queries(List *querytrees, const char *query_string, int cursorOptions, - ParamListInfo boundParams) { - List *stmt_list = NIL; - ListCell *query_list; - - foreach(query_list, querytrees) { - Query *query = lfirst_node(Query, query_list); - PlannedStmt *stmt; - if (query->commandType == CMD_UTILITY) { - stmt = makeNode(PlannedStmt); - stmt->commandType = CMD_UTILITY; - stmt->canSetTag = query->canSetTag; - stmt->utilityStmt = query->utilityStmt; - stmt->stmt_location = query->stmt_location; - stmt->stmt_len = query->stmt_len; - } else { - stmt = planner(query, query_string, cursorOptions, - boundParams); - } +static void +exec_simple_query(const char *query_string) +{ + MemoryContext oldcontext; + List *parsetree_list; + ListCell *parsetree_item; + bool use_implicit_block; + + StartTransactionCommand(); + oldcontext = MemoryContextSwitchTo(MessageContext); + + parsetree_list = raw_parser(query_string); + MemoryContextSwitchTo(oldcontext); + + use_implicit_block = (list_length(parsetree_list) > 1); + + foreach(parsetree_item, parsetree_list) + { + RawStmt *parsetree = lfirst_node(RawStmt, parsetree_item); + bool snapshot_set = false; + MemoryContext per_parsetree_context = NULL; + List *querytree_list, + *plantree_list; + + if (use_implicit_block) + BeginImplicitTransactionBlock(); + + if (analyze_requires_snapshot(parsetree)) + { + PushActiveSnapshot(GetTransactionSnapshot()); + snapshot_set = true; + } + + if (lnext(parsetree_list, parsetree_item) != NULL) + { + per_parsetree_context = + AllocSetContextCreate(MessageContext, + "per-parsetree message context", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(per_parsetree_context); + } + else + oldcontext = MemoryContextSwitchTo(MessageContext); + + querytree_list = pg_analyze_and_rewrite(parsetree, query_string, + NULL, 0, NULL); - stmt_list = lappend(stmt_list, stmt); - } + plantree_list = pg_plan_queries(querytree_list, query_string, + CURSOR_OPT_PARALLEL_OK, NULL); + + if (per_parsetree_context){ + MemoryContextDelete(per_parsetree_context); + } + CommitTransactionCommand(); + } +} + +static void fuzzer_exit(){ + if(!username) + pfree((void *) username); +} + + +int __attribute__((constructor)) Initialize(void) { + int argc = 4; + char *argv[4]; + argv[0] = "tmp_install/usr/local/pgsql/bin/postgres"; + argv[1] = "-D\"temp/data\""; + argv[2] = "-F"; + argv[3] = "-k\"/tmp/pg_dbfuzz\""; + + progname = get_progname(argv[0]); + MemoryContextInit(); + + username = strdup(get_user_name_or_exit(progname)); + + InitStandaloneProcess(argv[0]); + SetProcessingMode(InitProcessing); + InitializeGUCOptions(); + process_postgres_switches(argc, argv, PGC_POSTMASTER, &dbname); + dbname = "dbfuzz"; + + userDoption = "temp/data"; + SelectConfigFiles(userDoption, progname); + + checkDataDir(); + ChangeToDataDir(); + CreateDataDirLockFile(false); + LocalProcessControlFile(false); + InitializeMaxBackends(); + + BaseInit(); + InitProcess(); + PG_SETMASK(&UnBlockSig); + InitPostgres(dbname, InvalidOid, username, InvalidOid, NULL, false); - return stmt_list; - } + SetProcessingMode(NormalProcessing); + + BeginReportingGUCOptions(); + process_session_preload_libraries(); + + MessageContext = AllocSetContextCreate(TopMemoryContext, + "MessageContext", + ALLOCSET_DEFAULT_SIZES); + row_description_context = AllocSetContextCreate(TopMemoryContext, + "RowDescriptionContext", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(row_description_context); + initStringInfo(&row_description_buf); + MemoryContextSwitchTo(TopMemoryContext); + + PgStartTime = GetCurrentTimestamp(); + whereToSendOutput = DestNone; + Log_destination = 0; + atexit(fuzzer_exit); + return 0; +} /* @@ -60,64 +165,42 @@ List *plan_queries(List *querytrees, const char *query_string, int cursorOptions ** fuzzed input. */ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - char* query_string; - sigjmp_buf local_sigjmp_buf; - List *parsetree_list; - ListCell *parsetree_item; + char* query_string; + sigjmp_buf local_sigjmp_buf; - MemoryContextInit(); - query_string = (char*) calloc( (size+1), sizeof(char) ); - memcpy(query_string, data, size); - MessageContext = AllocSetContextCreate(TopMemoryContext, - "MessageContext", - ALLOCSET_DEFAULT_SIZES); + query_string = (char*) calloc( (size+1), sizeof(char) ); + memcpy(query_string, data, size); - if(!sigsetjmp(local_sigjmp_buf,0)){ - MemoryContext oldcontext; + if (!sigsetjmp(local_sigjmp_buf, 0)) + { + PG_exception_stack = &local_sigjmp_buf; + error_context_stack = NULL; - error_context_stack = NULL; - PG_exception_stack = &local_sigjmp_buf; + disable_all_timeouts(false); + QueryCancelPending = false; + pq_comm_reset(); + EmitErrorReport(); - oldcontext = MemoryContextSwitchTo(MessageContext); - parsetree_list = raw_parser(query_string); - MemoryContextSwitchTo(oldcontext); - - foreach(parsetree_item, parsetree_list) { - RawStmt *parsetree = lfirst_node(RawStmt, parsetree_item); - MemoryContext per_parsetree_context = NULL; - List *querytree_list; - Query *query; + AbortCurrentTransaction(); + + PortalErrorCleanup(); + SPICleanup(); - if (analyze_requires_snapshot(parsetree)){ - PushActiveSnapshot(GetTransactionSnapshot()); - } - if (lnext(parsetree_list, parsetree_item) != NULL){ - per_parsetree_context = - AllocSetContextCreate(MessageContext, - "per-parsetree message context", - ALLOCSET_DEFAULT_SIZES); - MemoryContextSwitchTo(per_parsetree_context); - } else { - MemoryContextSwitchTo(MessageContext); - } - query = parse_analyze(parsetree, query_string, NULL, 0, NULL); - if (query->commandType == CMD_UTILITY) { - querytree_list = list_make1(query); - } else { - querytree_list = QueryRewrite(query); - } - plan_queries(querytree_list, query_string, CURSOR_OPT_PARALLEL_OK, NULL); - if (per_parsetree_context){ - MemoryContextDelete(per_parsetree_context); - } - } - } - - free(query_string); - FlushErrorState(); - MemoryContextReset(TopMemoryContext); - TopMemoryContext->ident = NULL; - TopMemoryContext->methods->delete_context(TopMemoryContext); - VALGRIND_DESTROY_MEMPOOL(TopMemoryContext); - return 0; + jit_reset_after_error(); + + MemoryContextSwitchTo(TopMemoryContext); + FlushErrorState(); + + MemoryContextSwitchTo(MessageContext); + MemoryContextResetAndDeleteChildren(MessageContext); + + InvalidateCatalogSnapshotConditionally(); + + SetCurrentStatementStartTimestamp(); + + exec_simple_query(query_string); + } + + free(query_string); + return 0; } diff --git a/projects/postgresql/project.yaml b/projects/postgresql/project.yaml index d2ff7f56a..ce7df34b6 100644 --- a/projects/postgresql/project.yaml +++ b/projects/postgresql/project.yaml @@ -1,7 +1,7 @@ homepage: "https://postgresql.org" primary_contact: "sfrost@snowman.net" language: c -auto_ccs : +auto_ccs: - "ouyangyunshu@google.com" - "frost.stephen.p@gmail.com" fuzzing_engines: @@ -9,4 +9,3 @@ fuzzing_engines: - honggfuzz sanitizers: - address - - undefined