Merge branch 'fixes'

2019-02-13 23:51:46 +00:00 · 2019-02-13 23:51:46 +00:00 · a2d4535db8
commit a2d4535db8
parent 9378016b19 bb72cf9088
27 changed files with 387 additions and 117 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -11,7 +11,7 @@ project(mosquitto)
 cmake_minimum_required(VERSION 2.8)
 # Only for version 3 and up. cmake_policy(SET CMP0042 NEW)

-set (VERSION 1.5.6)
+set (VERSION 1.5.7)

 add_definitions (-DCMAKE -DVERSION=\"${VERSION}\")

--- a/ChangeLog.txt
+++ b/ChangeLog.txt
@ -1,8 +1,24 @@
-1.5.7 - 201902xx
+1.5.7 - 20190213
 ================

 Broker:
 - Fix build failure when using WITH_ADNS=yes
+- Ensure that an error occurs if `per_listener_settings true` is given after
+  other security options. Closes #1149.
+- Fix include_dir not sorting config files before loading. This was partially
+  fixed in 1.5 previously.
+- Improve documentation around the `include_dir` option. Closes #1154.
+- Fix case where old unreferenced msg_store messages were being saved to the
+  persistence file, bloating its size unnecessarily. Closes #389.
+
+Library:
+- Fix `mosquitto_topic_matches_sub()` not returning MOSQ_ERR_INVAL for
+  invalid subscriptions like `topic/#abc`. This only affects the return value,
+  not the match/no match result, which was already correct.
+
+Build:
+- Don't require C99 compiler.
+- Add rewritten build test script and remove some build warnings.


 1.5.6 - 20190206
--- a/buildtest.py
+++ b/buildtest.py
@ -0,0 +1,59 @@
+#!/usr/bin/python3
+
+build_variants = [
+    'WITH_ADNS',
+    'WITH_BRIDGE',
+    'WITH_DOCS',
+    'WITH_EC',
+    'WITH_EPOLL',
+    'WITH_MEMORY_TRACKING',
+    'WITH_PERSISTENCE',
+    'WITH_SHARED_LIBRARIES',
+    'WITH_SOCKS',
+    'WITH_SRV',
+    'WITH_STATIC_LIBRARIES',
+    'WITH_STRIP',
+    'WITH_SYSTEMD',
+    'WITH_SYS_TREE',
+    'WITH_THREADING',
+    'WITH_TLS',
+    'WITH_TLS_PSK',
+    'WITH_WEBSOCKETS',
+    'WITH_WRAP',
+]
+
+special_variants = [
+    'WITH_BUNDLED_DEPS',
+    'WITH_COVERAGE',
+]
+
+
+import random
+import subprocess
+
+def run_test(msg, opts):
+    subprocess.run(["make", "clean"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    print("%s: %s" % (msg, str(opts)))
+    args = ["make", "-j"] + opts
+    proc = subprocess.run(args, stdout=subprocess.DEVNULL)
+    if proc.returncode != 0:
+        raise RuntimeError("BUILD FAILED: %s" % (' '.join(args)))
+    
+def simple_tests():
+    for bv in build_variants:
+        for enabled in ["yes", "no"]:
+            opts = "%s=%s" % (bv, enabled)
+            run_test("SIMPLE BUILD", [opts])
+
+def random_tests(count=10):
+    for i in range(1, count):
+        opts = []
+        for bv in build_variants:
+            opts.append("%s=%s" % (bv, random.choice(["yes", "no"])))
+
+        run_test("RANDOM BUILD", opts)
+
+
+if __name__ == "__main__":
+    simple_tests()
+    random_tests(100)
--- a/client/Makefile
+++ b/client/Makefile
@ -21,10 +21,10 @@ static : static_pub static_sub
 	# libmosquitto only.

 static_pub : pub_client.o client_shared.o ../lib/libmosquitto.a
-	${CROSS_COMPILE}${CC} $^ -o mosquitto_pub ${CLIENT_LDFLAGS} -lssl -lcrypto -lpthread
+	${CROSS_COMPILE}${CC} $^ -o mosquitto_pub ${CLIENT_LDFLAGS} ${STATIC_LIB_DEPS}

 static_sub : sub_client.o sub_client_output.o client_shared.o ../lib/libmosquitto.a
-	${CROSS_COMPILE}${CC} $^ -o mosquitto_sub ${CLIENT_LDFLAGS} -lssl -lcrypto -lpthread
+	${CROSS_COMPILE}${CC} $^ -o mosquitto_sub ${CLIENT_LDFLAGS} ${STATIC_LIB_DEPS}

 mosquitto_pub : pub_client.o client_shared.o
 	${CROSS_COMPILE}${CC} $^ -o $@ ${CLIENT_LDFLAGS}
--- a/config.mk
+++ b/config.mk
@ -105,7 +105,7 @@ WITH_BUNDLED_DEPS:=yes

 # Also bump lib/mosquitto.h, CMakeLists.txt,
 # installer/mosquitto.nsi, installer/mosquitto64.nsi
-VERSION=1.5.6
+VERSION=1.5.7

 # Client library SO version. Bump if incompatible API/ABI changes are made.
 SOVERSION=1
@ -129,6 +129,7 @@ else
 	CFLAGS?=-Wall -ggdb -O2
 endif

+STATIC_LIB_DEPS:=
 LIB_CFLAGS:=${CFLAGS} ${CPPFLAGS} -I. -I.. -I../lib
 LIB_CXXFLAGS:=$(CFLAGS) ${CPPFLAGS} -I. -I.. -I../lib
 LIB_LDFLAGS:=${LDFLAGS}
@ -192,6 +193,7 @@ ifeq ($(WITH_TLS),yes)
 	LIB_CFLAGS:=$(LIB_CFLAGS) -DWITH_TLS
 	PASSWD_LIBS:=-lcrypto
 	CLIENT_CFLAGS:=$(CLIENT_CFLAGS) -DWITH_TLS
+	STATIC_LIB_DEPS:=$(STATIC_LIB_DEPS) -lssl -lcrypto

 	ifeq ($(WITH_TLS_PSK),yes)
 		BROKER_CFLAGS:=$(BROKER_CFLAGS) -DWITH_TLS_PSK
@ -204,6 +206,7 @@ ifeq ($(WITH_THREADING),yes)
 	LIB_LIBS:=$(LIB_LIBS) -lpthread
 	LIB_CFLAGS:=$(LIB_CFLAGS) -DWITH_THREADING
 	CLIENT_CFLAGS:=$(CLIENT_CFLAGS) -DWITH_THREADING
+	STATIC_LIB_DEPS:=$(STATIC_LIB_DEPS) -lpthread
 endif

 ifeq ($(WITH_SOCKS),yes)
@ -249,6 +252,7 @@ ifeq ($(WITH_SRV),yes)
 	LIB_CFLAGS:=$(LIB_CFLAGS) -DWITH_SRV
 	LIB_LIBS:=$(LIB_LIBS) -lcares
 	CLIENT_CFLAGS:=$(CLIENT_CFLAGS) -DWITH_SRV
+	STATIC_LIB_DEPS:=$(STATIC_LIB_DEPS) -lcares
 endif

 ifeq ($(UNAME),SunOS)
--- a/installer/mosquitto.nsi
+++ b/installer/mosquitto.nsi
@ -9,7 +9,7 @@
 !define env_hklm 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"'

 Name "Eclipse Mosquitto"
-!define VERSION 1.5.6
+!define VERSION 1.5.7
 OutFile "mosquitto-${VERSION}-install-windows-x86.exe"

 InstallDir "$PROGRAMFILES\mosquitto"
--- a/installer/mosquitto64.nsi
+++ b/installer/mosquitto64.nsi
@ -9,7 +9,7 @@
 !define env_hklm 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"'

 Name "Eclipse Mosquitto"
-!define VERSION 1.5.6
+!define VERSION 1.5.7
 OutFile "mosquitto-${VERSION}-install-windows-x64.exe"

 !include "x64.nsh"
--- a/lib/mosquitto.h
+++ b/lib/mosquitto.h
@ -47,7 +47,7 @@ extern "C" {

 #define LIBMOSQUITTO_MAJOR 1
 #define LIBMOSQUITTO_MINOR 5
-#define LIBMOSQUITTO_REVISION 6
+#define LIBMOSQUITTO_REVISION 7
 /* LIBMOSQUITTO_VERSION_NUMBER looks like 1002001 for e.g. version 1.2.1. */
 #define LIBMOSQUITTO_VERSION_NUMBER (LIBMOSQUITTO_MAJOR*1000000+LIBMOSQUITTO_MINOR*1000+LIBMOSQUITTO_REVISION)

--- a/lib/util_mosq.c
+++ b/lib/util_mosq.c
@ -245,6 +245,7 @@ int mosquitto_topic_matches_sub2(const char *sub, size_t sublen, const char *top
 {
 	int spos, tpos;
 	bool multilevel_wildcard = false;
+	int i;

 	if(!result) return MOSQ_ERR_INVAL;
 	*result = false;
@ -270,31 +271,10 @@ int mosquitto_topic_matches_sub2(const char *sub, size_t sublen, const char *top
 	tpos = 0;

 	while(spos < sublen && tpos <= topiclen){
-		if(sub[spos] == topic[tpos]){
-			if(tpos == topiclen-1){
-				/* Check for e.g. foo matching foo/# */
-				if(spos == sublen-3
-						&& sub[spos+1] == '/'
-						&& sub[spos+2] == '#'){
-					*result = true;
-					multilevel_wildcard = true;
-					return MOSQ_ERR_SUCCESS;
-				}
-			}
-			spos++;
-			tpos++;
-			if(spos == sublen && tpos == topiclen){
-				*result = true;
-				return MOSQ_ERR_SUCCESS;
-			}else if(tpos == topiclen && spos == sublen-1 && sub[spos] == '+'){
-				if(spos > 0 && sub[spos-1] != '/'){
-					return MOSQ_ERR_INVAL;
-				}
-				spos++;
-				*result = true;
-				return MOSQ_ERR_SUCCESS;
-			}
-		}else{
+		if(topic[tpos] == '+' || topic[tpos] == '#'){
+			return MOSQ_ERR_INVAL;
+		}
+		if(tpos == topiclen || sub[spos] != topic[tpos]){ /* Check for wildcard matches */
 			if(sub[spos] == '+'){
 				/* Check for bad "+foo" or "a/+foo" subscription */
 				if(spos > 0 && sub[spos-1] != '/'){
@ -336,6 +316,39 @@ int mosquitto_topic_matches_sub2(const char *sub, size_t sublen, const char *top
 					multilevel_wildcard = true;
 					return MOSQ_ERR_SUCCESS;
 				}
+
+				for(i=spos; i<sublen; i++){
+					if(sub[i] == '#' && i+1 != sublen){
+						return MOSQ_ERR_INVAL;
+					}
+				}
+
+				/* Valid input, but no match */
+				return MOSQ_ERR_SUCCESS;
+			}
+		}else{
+			/* sub[spos] == topic[tpos] */
+			if(tpos == topiclen-1){
+				/* Check for e.g. foo matching foo/# */
+				if(spos == sublen-3
+						&& sub[spos+1] == '/'
+						&& sub[spos+2] == '#'){
+					*result = true;
+					multilevel_wildcard = true;
+					return MOSQ_ERR_SUCCESS;
+				}
+			}
+			spos++;
+			tpos++;
+			if(spos == sublen && tpos == topiclen){
+				*result = true;
+				return MOSQ_ERR_SUCCESS;
+			}else if(tpos == topiclen && spos == sublen-1 && sub[spos] == '+'){
+				if(spos > 0 && sub[spos-1] != '/'){
+					return MOSQ_ERR_INVAL;
+				}
+				spos++;
+				*result = true;
 				return MOSQ_ERR_SUCCESS;
 			}
 		}
--- a/man/mosquitto.conf.5.xml
+++ b/man/mosquitto.conf.5.xml
@ -349,6 +349,55 @@
 						file. This option will only be processed from the main
 						configuration file. The directory specified must not
 						contain the main configuration file.</para>
+					<para>The configuration files in
+						<option>include_dir</option> are loaded in case
+						sensitive alphabetical order, with the upper case of
+						each letter ordered before the lower case of the same
+						letter.</para>
+						<example title="Load Order for include_dir" label="Load Order for include_dir">
+							<para>Given the files
+								<replaceable>b.conf</replaceable>,
+								<replaceable>A.conf</replaceable>,
+								<replaceable>01.conf</replaceable>,
+								<replaceable>a.conf</replaceable>,
+								<replaceable>B.conf</replaceable>, and
+								<replaceable>00.conf</replaceable> inside
+								<option>include_dir</option>, the config files
+								would be loaded in this order:</para>
+						<programlisting language="config">
+00.conf
+01.conf
+A.conf
+a.conf
+B.conf
+b.conf
+</programlisting></example>
+					<para>If this option is used multiple times, then each
+						<option>include_dir</option> option is processed
+						completely in the order that they are written in the
+						main configuration file.</para>
+						<example title="Load Order for Multiple include_dir" label="Load Order for Multiple include_dir">
+							<para>Assuming a directory
+								<replaceable>one.d</replaceable> containing
+								files <replaceable>B.conf</replaceable> and
+								<replaceable>C.conf</replaceable>, and a second
+								directory <replaceable>two.d</replaceable>
+								containing files
+								<replaceable>A.conf</replaceable> and
+								<replaceable>D.conf</replaceable>, and a
+								config:</para>
+						<programlisting language="config">
+include_dir one.d
+include_dir two.d
+</programlisting><para>Then the config files would be loaded in this order:</para>
+						<programlisting language="config">
+# files from one.d
+B.conf
+C.conf
+# files from two.d
+A.conf
+D.conf
+</programlisting></example>
 				</listitem>
 			</varlistentry>
 			<varlistentry>
--- a/mosquitto.conf
+++ b/mosquitto.conf
@ -872,6 +872,10 @@
 # in the main file. This option will only be processed from the main
 # configuration file. The directory specified must not contain the 
 # main configuration file.
+# Files within include_dir will be loaded sorted in case-sensitive
+# alphabetical order, with capital letters ordered first. If this option is
+# given multiple times, all of the files from the first instance will be
+# processed before the next instance. See the man page for examples.
 #include_dir

 # =================================================================
--- a/set-version.sh
+++ b/set-version.sh
@ -2,7 +2,7 @@

 MAJOR=1
 MINOR=5
-REVISION=6
+REVISION=7

 sed -i "s/^VERSION=.*/VERSION=${MAJOR}.${MINOR}.${REVISION}/" config.mk

--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@ -1,5 +1,5 @@
 name: mosquitto
-version: 1.5.6
+version: 1.5.7
 summary: Eclipse Mosquitto MQTT broker
 description: This is a message broker that supports version 3.1 and 3.1.1 of the MQTT
    protocol.
--- a/src/bridge.c
+++ b/src/bridge.c
@ -112,7 +112,6 @@ int bridge__new(struct mosquitto_db *db, struct mosquitto__bridge *bridge)
 int bridge__connect_step1(struct mosquitto_db *db, struct mosquitto *context)
 {
 	int rc;
-	int i;
 	char *notification_topic;
 	int notification_topic_len;
 	uint8_t notification_payload;
--- a/src/conf.c
+++ b/src/conf.c
@ -62,6 +62,8 @@ struct config_recurse {
 extern SERVICE_STATUS_HANDLE service_handle;
 #endif

+static struct mosquitto__security_options *cur_security_options = NULL;
+
 static int conf__parse_bool(char **token, const char *name, bool *value, char *saveptr);
 static int conf__parse_int(char **token, const char *name, int *value, char *saveptr);
 static int conf__parse_ssize_t(char **token, const char *name, ssize_t *value, char *saveptr);
@ -725,25 +727,15 @@ int config__read_file_core(struct mosquitto__config *config, bool reload, struct
 	char *tmp_char;
 	struct mosquitto__bridge *cur_bridge = NULL;
 	struct mosquitto__bridge_topic *cur_topic;
+	int len;
 #endif
 	struct mosquitto__auth_plugin_config *cur_auth_plugin_config = NULL;

 	time_t expiration_mult;
 	char *key;
-	char *conf_file;
-#ifdef WIN32
-	HANDLE fh;
-	char dirpath[MAX_PATH];
-	WIN32_FIND_DATA find_data;
-#else
-	DIR *dh;
-	struct dirent *de;
-#endif
-	int len;
 	struct mosquitto__listener *cur_listener = &config->default_listener;
 	int i;
 	int lineno_ext;
-	struct mosquitto__security_options *cur_security_options = NULL;

 	*lineno = 0;

@ -1218,66 +1210,27 @@ int config__read_file_core(struct mosquitto__config *config, bool reload, struct
 							log__printf(NULL, MOSQ_LOG_ERR, "Error: Empty include_dir value in configuration.");
 							return 1;
 						}
-#ifdef WIN32
-						snprintf(dirpath, MAX_PATH, "%s\\*.conf", token);
-						fh = FindFirstFile(dirpath, &find_data);
-						if(fh == INVALID_HANDLE_VALUE){
-							/* No files found */
-							continue;
-						}

-						do{
-							len = strlen(token)+1+strlen(find_data.cFileName)+1;
-							conf_file = mosquitto__malloc(len+1);
-							if(!conf_file){
-								FindClose(fh);
-								return MOSQ_ERR_NOMEM;
-							}
-							snprintf(conf_file, len, "%s\\%s", token, find_data.cFileName);
-							conf_file[len] = '\0';
+						char **files;
+						int file_count;
+						rc = config__get_dir_files(token, &files, &file_count);
+						if(rc) return rc;

-							rc = config__read_file(config, reload, conf_file, cr, level+1, &lineno_ext);
+						for(i=0; i<file_count; i++){
+							log__printf(NULL, MOSQ_LOG_INFO, "Loading config file %s", files[i]);
+
+							rc = config__read_file(config, reload, files[i], cr, level+1, &lineno_ext);
 							if(rc){
-								FindClose(fh);
-								log__printf(NULL, MOSQ_LOG_ERR, "Error found at %s:%d.", conf_file, lineno_ext);
-								mosquitto__free(conf_file);
-								return rc;
-							}
-							mosquitto__free(conf_file);
-						}while(FindNextFile(fh, &find_data));
-
-						FindClose(fh);
-#else
-						dh = opendir(token);
-						if(!dh){
-							log__printf(NULL, MOSQ_LOG_ERR, "Error: Unable to open include_dir '%s'.", token);
-							return 1;
-						}
-						while((de = readdir(dh)) != NULL){
-							if(strlen(de->d_name) > 5){
-								if(!strcmp(&de->d_name[strlen(de->d_name)-5], ".conf")){
-									len = strlen(token)+1+strlen(de->d_name)+1;
-									conf_file = mosquitto__malloc(len+1);
-									if(!conf_file){
-										closedir(dh);
-										return MOSQ_ERR_NOMEM;
-									}
-									snprintf(conf_file, len, "%s/%s", token, de->d_name);
-									conf_file[len] = '\0';
-
-									rc = config__read_file(config, reload, conf_file, cr, level+1, &lineno_ext);
-									if(rc){
-										closedir(dh);
-										log__printf(NULL, MOSQ_LOG_ERR, "Error found at %s:%d.", conf_file, lineno_ext);
-										mosquitto__free(conf_file);
-										return rc;
-									}
-									mosquitto__free(conf_file);
-								}
+								log__printf(NULL, MOSQ_LOG_ERR, "Error found at %s:%d.", files[i], lineno_ext);
+								/* Free happens below */
+								break;
 							}
 						}
-						closedir(dh);
-#endif
+						for(i=0; i<file_count; i++){
+							mosquitto__free(files[i]);
+						}
+						mosquitto__free(files);
+						if(rc) return rc; /* This returns if config__read_file() fails above */
 					}
 				}else if(!strcmp(token, "keepalive_interval")){
 #ifdef WITH_BRIDGE
--- a/src/conf_includedir.c
+++ b/src/conf_includedir.c
@ -16,6 +16,7 @@ Contributors:

 #include "config.h"

+#include <ctype.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
@ -46,11 +47,32 @@ Contributors:
 #include "util_mosq.h"
 #include "mqtt3_protocol.h"

-int strcasecmp_p(const void *p1, const void *p2)
-{
-	return strcasecmp(*(const char **)p1, *(const char **)p2);
-}

+int scmp_p(const void *p1, const void *p2)
+{
+	const char *s1 = *(const char **)p1;
+	const char *s2 = *(const char **)p2;
+	int result;
+
+	while(s1[0] && s2[0]){
+		/* Sort by case insensitive part first */
+		result = toupper(s1[0]) - toupper(s2[0]);
+		if(result == 0){
+			/* Case insensitive part matched, now distinguish between case */
+			result = s1[0] - s2[0];
+			if(result != 0){
+				return result;
+			}
+		}else{
+			/* Return case insensitive match fail */
+			return result;
+		}
+		s1++;
+		s2++;
+	}
+
+	return s1[0] - s2[0];
+}

 #ifdef WIN32
 int config__get_dir_files(const char *include_dir, char ***files, int *file_count)
@ -102,6 +124,7 @@ int config__get_dir_files(const char *include_dir, char ***files, int *file_coun

 	FindClose(fh);

+	qsort(l_files, l_file_count, sizeof(char *), scmp_p);
 	*files = l_files;
 	*file_count = l_file_count;

@ -111,6 +134,7 @@ int config__get_dir_files(const char *include_dir, char ***files, int *file_coun


 #ifndef WIN32
+
 int config__get_dir_files(const char *include_dir, char ***files, int *file_count)
 {
 	char **l_files = NULL;
@ -160,6 +184,7 @@ int config__get_dir_files(const char *include_dir, char ***files, int *file_coun
 	}
 	closedir(dh);

+	qsort(l_files, l_file_count, sizeof(char *), scmp_p);
 	*files = l_files;
 	*file_count = l_file_count;

--- a/src/database.c
+++ b/src/database.c
@ -235,6 +235,21 @@ void db__msg_store_deref(struct mosquitto_db *db, struct mosquitto_msg_store **s
 }


+void db__msg_store_compact(struct mosquitto_db *db)
+{
+	struct mosquitto_msg_store *store, *next;
+
+	store = db->msg_store;
+	while(store){
+		next = store->next;
+		if(store->ref_count < 1){
+			db__msg_store_remove(db, store);
+		}
+		store = next;
+	}
+}
+
+
 static void db__message_remove(struct mosquitto_db *db, struct mosquitto *context, struct mosquitto_client_msg **msg, struct mosquitto_client_msg *last)
 {
 	if(!context || !msg || !(*msg)){
@ -1019,8 +1034,3 @@ void db__limits_set(int inflight, unsigned long inflight_bytes, int queued, unsi
 	max_queued_bytes = queued_bytes;
 }

-void db__vacuum(void)
-{
-	/* FIXME - reimplement? */
-}
-
--- a/src/loop.c
+++ b/src/loop.c
@ -756,7 +756,7 @@ static void loop_handle_reads_writes(struct mosquitto_db *db, struct pollfd *pol
 				if(!getsockopt(context->sock, SOL_SOCKET, SO_ERROR, (char *)&err, &len)){
 					if(err == 0){
 						context->state = mosq_cs_new;
-#ifdef WITH_ADNS
+#if defined(WITH_ADNS) && defined(WITH_BRIDGE)
 						if(context->bridge){
 							bridge__connect_step3(db, context);
 							continue;
--- a/src/mosquitto_broker_internal.h
+++ b/src/mosquitto_broker_internal.h
@ -562,8 +562,8 @@ void db__msg_store_add(struct mosquitto_db *db, struct mosquitto_msg_store *stor
 void db__msg_store_remove(struct mosquitto_db *db, struct mosquitto_msg_store *store);
 void db__msg_store_deref(struct mosquitto_db *db, struct mosquitto_msg_store **store);
 void db__msg_store_clean(struct mosquitto_db *db);
+void db__msg_store_compact(struct mosquitto_db *db);
 int db__message_reconnect_reset(struct mosquitto_db *db, struct mosquitto *context);
-void db__vacuum(void);
 void sys_tree__init(struct mosquitto_db *db);
 void sys_tree__update(struct mosquitto_db *db, int interval, time_t start_time);

--- a/src/persist.c
+++ b/src/persist.c
@ -151,6 +151,11 @@ static int persist__message_store_write(struct mosquitto_db *db, FILE *db_fptr)

 	stored = db->msg_store;
 	while(stored){
+		if(stored->ref_count < 1){
+			stored = stored->next;
+			continue;
+		}
+
 		if(stored->topic && !strncmp(stored->topic, "$SYS", 4)){
 			if(stored->ref_count <= 1 && stored->dest_id_count == 0){
 				/* $SYS messages that are only retained shouldn't be persisted. */
@ -720,6 +725,7 @@ static int persist__msg_store_chunk_restore(struct mosquitto_db *db, FILE *db_fp
 	struct mosquitto_msg_store *stored = NULL;
 	struct mosquitto_msg_store_load *load;
 	char *err;
+	int i;

 	payload.ptr = NULL;

@ -749,7 +755,7 @@ static int persist__msg_store_chunk_restore(struct mosquitto_db *db, FILE *db_fp
 		read_e(db_fptr, &i16temp, sizeof(uint16_t));
 		source_port = ntohs(i16temp);
 		if(source_port){
-			for(int i=0; i<db->config->listener_count; i++){
+			for(i=0; i<db->config->listener_count; i++){
 				if(db->config->listeners[i].port == source_port){
 					source.listener = &db->config->listeners[i];
 					break;
@ -981,6 +987,9 @@ int persist__restore(struct mosquitto_db *db)
 		HASH_DELETE(hh, db->msg_store_load, load);
 		mosquitto__free(load);
 	}
+
+	db__msg_store_compact(db);
+
 	return rc;
 error:
 	err = strerror(errno);
--- a/test/broker/03-pattern-matching.py
+++ b/test/broker/03-pattern-matching.py
@ -69,6 +69,7 @@ pattern_test("foo/+/baz/#", "foo/bar/baz")
 pattern_test("foo/+/baz/#", "foo/bar/baz/bar")
 pattern_test("foo/foo/baz/#", "foo/foo/baz/bar")
 pattern_test("foo/#", "foo")
+pattern_test("foo/#", "foo/")
 pattern_test("/#", "/foo")
 pattern_test("test/topic/", "test/topic/")
 pattern_test("test/topic/+", "test/topic/")
--- a/test/broker/08-ssl-connect-cert-auth-expired.py
+++ b/test/broker/08-ssl-connect-cert-auth-expired.py
@ -47,6 +47,7 @@ try:
    ssock.settimeout(20)
    try:
        ssock.connect(("localhost", port1))
+        mosq_test.do_send_receive(ssock, connect_packet, "", "connack")
    except ssl.SSLError as err:
        if err.errno == 1:
            rc = 0
--- a/test/broker/08-ssl-connect-cert-auth-revoked.py
+++ b/test/broker/08-ssl-connect-cert-auth-revoked.py
@ -44,6 +44,7 @@ try:
    ssock.settimeout(20)
    try:
        ssock.connect(("localhost", port1))
+        mosq_test.do_send_receive(ssock, connect_packet, "", "connack")
    except ssl.SSLError as err:
        if err.errno == 1 and "certificate revoked" in err.strerror:
            rc = 0
--- a/test/broker/08-ssl-connect-cert-auth-without.py
+++ b/test/broker/08-ssl-connect-cert-auth-without.py
@ -45,6 +45,7 @@ ssock = ssl.wrap_socket(sock, ca_certs="../ssl/test-root-ca.crt", cert_reqs=ssl.
 ssock.settimeout(20)
 try:
    ssock.connect(("localhost", port1))
+    mosq_test.do_send_receive(ssock, connect_packet, "", "connack")
 except ssl.SSLError as err:
    if err.errno == 1:
        rc = 0
--- a/www/pages/download.md
+++ b/www/pages/download.md
@ -11,7 +11,7 @@

 # Source

-* [mosquitto-1.5.5.tar.gz](https://mosquitto.org/files/source/mosquitto-1.5.5.tar.gz) (319kB) ([GPG signature](https://mosquitto.org/files/source/mosquitto-1.5.5.tar.gz.asc))
+* [mosquitto-1.5.7.tar.gz](https://mosquitto.org/files/source/mosquitto-1.5.7.tar.gz) (319kB) ([GPG signature](https://mosquitto.org/files/source/mosquitto-1.5.7.tar.gz.asc))
 * [Git source code repository](https://github.com/eclipse/mosquitto) (github.com)

 Older downloads are available at [https://mosquitto.org/files/](../files/)
@ -24,8 +24,8 @@ distributions.

 ## Windows

-* [mosquitto-1.5.5-install-windows-x64.exe](https://mosquitto.org/files/binary/win64/mosquitto-1.5.5-install-windows-x64.exe) (~360 kB) (64-bit build, Windows Vista and up, built with Visual Studio Community 2017)
-* [mosquitto-1.5.5-install-windows-x32.exe](https://mosquitto.org/files/binary/win32/mosquitto-1.5.5-install-windows-x86.exe) (~360 kB) (32-bit build, Windows Vista and up, built with Visual Studio Community 2017)
+* [mosquitto-1.5.6-install-windows-x64.exe](https://mosquitto.org/files/binary/win64/mosquitto-1.5.6-install-windows-x64.exe) (~360 kB) (64-bit build, Windows Vista and up, built with Visual Studio Community 2017)
+* [mosquitto-1.5.6-install-windows-x32.exe](https://mosquitto.org/files/binary/win32/mosquitto-1.5.6-install-windows-x86.exe) (~360 kB) (32-bit build, Windows Vista and up, built with Visual Studio Community 2017)

 See also readme-windows.txt after installing.

--- a/www/posts/2019/02/version-1-5-6-released.md
+++ b/www/posts/2019/02/version-1-5-6-released.md
@ -0,0 +1,89 @@
+<!--
+.. title: Version 1.5.6 released
+.. slug: version-1-5-6-released
+.. date: 2019-02-08 13:00:00 UTC
+.. tags: Security,Releases
+.. category:
+.. link:
+.. description:
+.. type: text
+-->
+
+Mosquitto 1.5.6 has been released to address three potential security vulnerabilities.
+
+# CVE-2018-12551
+
+If Mosquitto is configured to use a password file for authentication, any
+malformed data in the password file will be treated as valid. This typically
+means that the malformed data becomes a username and no password. If this
+occurs, clients can circumvent authentication and get access to the broker by
+using the malformed username. In particular, a blank line will be treated as a
+valid empty username. Other security measures are unaffected. **Users who have
+only used the `mosquitto_passwd` utility to create and modify their password
+files are unaffected by this vulnerability**. Affects version 1.0 to 1.5.5
+inclusive.
+
+Patches for older versions are available at <https://mosquitto.org/files/cve/2018-12551>
+
+# CVE-2018-12550
+
+If an ACL file is empty, or has only blank lines or comments, then mosquitto
+treats the ACL file as not being defined, which means that no topic access is
+denied. Although denying access to all topics is not a useful configuration,
+this behaviour is unexpected and could lead to access being incorrectly granted
+in some circumstances. Affects versions 1.0 to 1.5.5 inclusive.
+
+Patches for older versions are available at <https://mosquitto.org/files/cve/2018-12550>
+
+# CVE-2018-12546
+
+If a client publishes a retained message to a topic that they have access to,
+and then their access to that topic is revoked, the retained message will still
+be delivered to future subscribers. This behaviour may be undesirable in some
+applications, so a configuration option `check_retain_source` has been
+introduced to enforce checking of the retained message source on publish.  
+
+Patches for older versions are available at <https://mosquitto.org/files/cve/2018-12546>
+
+# Version 1.5.6 Changes
+
+The list of other fixes addressed in version 1.5.6 is:
+
+## Broker
+
+- Fixed comment handling for config options that have optional arguments.
+- Improved documentation around bridge topic remapping.
+- Handle mismatched handshakes (e.g. QoS1 PUBLISH with QoS2 reply) properly.
+- Fix spaces not being allowed in the bridge `remote_username option`. Closes
+  [#1131].
+- Allow broker to always restart on Windows when using `log_dest file`. Closes
+  [#1080].
+- Fix Will not being sent for Websockets clients. Closes [#1143].
+- Windows: Fix possible crash when client disconnects. Closes [#1137].
+- Fixed durable clients being unable to receive messages when offline, when
+  `per_listener_settings` was set to true. Closes [#1081].
+- Add log message for the case where a client is disconnected for sending a
+  topic with invalid UTF-8. Closes [#1144].
+
+## Library
+
+- Fix TLS connections not working over SOCKS.
+- Don't clear SSL context when TLS connection is closed, meaning if a user
+  provided an external SSL_CTX they have less chance of leaking references.
+
+## Build
+
+- Fix comparison of boolean values in CMake build. Closes [#1101].
+- Fix compilation when openssl deprecated APIs are not available.
+  Closes [#1094].
+- Man pages can now be built on any system. Closes [#1139].
+
+[#1080]: https://github.com/eclipse/mosquitto/issues/1080
+[#1081]: https://github.com/eclipse/mosquitto/issues/1081
+[#1094]: https://github.com/eclipse/mosquitto/issues/1094
+[#1101]: https://github.com/eclipse/mosquitto/issues/1101
+[#1131]: https://github.com/eclipse/mosquitto/issues/1131
+[#1137]: https://github.com/eclipse/mosquitto/issues/1137
+[#1139]: https://github.com/eclipse/mosquitto/issues/1139
+[#1143]: https://github.com/eclipse/mosquitto/issues/1143
+[#1144]: https://github.com/eclipse/mosquitto/issues/1144
--- a/www/posts/2019/02/version-1-5-7-released.md
+++ b/www/posts/2019/02/version-1-5-7-released.md
@ -0,0 +1,36 @@
+<!--
+.. title: Version 1.5.7 released
+.. slug: version-1-5-7-released
+.. date: 2019-02-13 23:50:00 UTC
+.. tags: Releases
+.. category:
+.. link:
+.. description:
+.. type: text
+-->
+
+This is a bugfix release.
+
+## Broker
+- Fix build failure when using `WITH_ADNS=yes`
+- Ensure that an error occurs if `per_listener_settings true` is given after
+  other security options. Closes [#1149].
+- Fix `include_dir` not sorting config files before loading. This was partially
+  fixed in 1.5 previously.
+- Improve documentation around the `include_dir` option. Closes [#1154].
+- Fix case where old unreferenced msg_store messages were being saved to the
+  persistence file, bloating its size unnecessarily. Closes [#389].
+
+## Library
+- Fix `mosquitto_topic_matches_sub()` not returning MOSQ_ERR_INVAL for
+  invalid subscriptions like `topic/#abc`. This only affects the return value,
+  not the match/no match result, which was already correct.
+
+## Build
+- Don't require C99 compiler.
+- Add rewritten build test script and remove some build warnings.
+
+
+[#389]: https://github.com/eclipse/mosquitto/issues/389
+[#1149]: https://github.com/eclipse/mosquitto/issues/1149
+[#1154]: https://github.com/eclipse/mosquitto/issues/1154