Add option to detect available cores when num_threads=-1 (#380)

When configuring `num_threads=-1` in `renderd.conf` for a `renderd` section, the value will be set to the number of cores on the running system.

Author: hummeltech

docs/man/renderd.conf.5

@@ -1,4 +1,4 @@
-.TH RENDERD.CONF 5 "2023-12-19" "mod_tile v0.7.0"
+.TH RENDERD.CONF 5 "2024-01-22" "mod_tile v0.7.0"
 .\" Please adjust this date whenever revising the manpage.
 
 .SH NAME
@@ -41,6 +41,7 @@ This option and \fBsocketname\fR are mutually exclusive.
 .TP
 .B num_threads
 Specify the number of threads to be used for \fBrenderd\fR.
+A value of \fB'-1'\fR will configure \fBnum_threads\fR to the number of cores on the system.
 The default value is \fB'4'\fR (macro definition \fB'NUM_THREADS'\fR).
 
 .TP
@@ -205,4 +206,4 @@ Only used by \fBrenderd\fR.
 .SH AUTHOR
 renderd was written by Jon Burgess, and other OpenStreetMap project members.
 .PP
-This manual page was written by OpenStreetMap authors.
+This manual page was written by OpenStreetMap authors.

src/renderd.c

@@ -66,7 +66,7 @@ static int exit_pipe_fd;
 
 static renderd_config config;
 
-int noSlaveRenders;
+int num_slave_threads;
 
 int foreground = 0;
 
@@ -826,7 +826,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	noSlaveRenders = 0;
+	num_slave_threads = 0;
 
 	int iconf = -1;
 	char buffer[PATH_MAX];
@@ -873,6 +873,10 @@ int main(int argc, char **argv)
 			config_slaves[render_sec].pid_filename = iniparser_getstring(ini,
 					buffer, (char *) RENDERD_PIDFILE);
 
+			if (config_slaves[render_sec].num_threads == -1) {
+				config_slaves[render_sec].num_threads = sysconf(_SC_NPROCESSORS_ONLN);
+			}
+
 			if (render_sec == active_slave) {
 				config.socketname = config_slaves[render_sec].socketname;
 				config.iphostname = config_slaves[render_sec].iphostname;
@@ -890,7 +894,7 @@ int main(int argc, char **argv)
 				config.mapnik_font_dir_recurse = iniparser_getboolean(ini,
 								 "mapnik:font_dir_recurse", MAPNIK_FONTS_DIR_RECURSE);
 			} else {
-				noSlaveRenders += config_slaves[render_sec].num_threads;
+				num_slave_threads += config_slaves[render_sec].num_threads;
 			}
 		}
 	}
@@ -1051,7 +1055,7 @@ int main(int argc, char **argv)
 	g_logger(G_LOG_LEVEL_INFO, "config renderd: num_threads=%d", config.num_threads);
 
 	if (active_slave == 0) {
-		g_logger(G_LOG_LEVEL_INFO, "config renderd: num_slaves=%d", noSlaveRenders);
+		g_logger(G_LOG_LEVEL_INFO, "config renderd: num_slave_threads=%d", num_slave_threads);
 	}
 
 	g_logger(G_LOG_LEVEL_INFO, "config renderd: tile_dir=%s", config.tile_dir);
@@ -1165,7 +1169,7 @@ int main(int argc, char **argv)
 		//Only the master renderd opens connections to its slaves
 		k = 0;
 		slave_threads
-			= (pthread_t *) malloc(sizeof(pthread_t) * noSlaveRenders);
+			= (pthread_t *) malloc(sizeof(pthread_t) * num_slave_threads);
 
 		for (i = 1; i < MAX_SLAVES; i++) {
 			for (j = 0; j < config_slaves[i].num_threads; j++) {

tests/CMakeLists.txt

@@ -11,6 +11,9 @@
 #-----------------------------------------------------------------------------
 
 include(CTest)
+include(ProcessorCount)
+
+processorcount(PROCESSOR_COUNT)
 
 execute_process(COMMAND ${APXS_EXECUTABLE} -q progname
   OUTPUT_VARIABLE HTTPD_PROGNAME
@@ -635,6 +638,28 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH})
   set_tests_properties(stats_urls_${STORAGE_BACKEND} PROPERTIES
     FIXTURES_REQUIRED "services_started_${STORAGE_BACKEND};tiles_downloaded_${STORAGE_BACKEND}"
   )
+  if(NOT PROCESSOR_COUNT EQUAL 0)
+    # Set CTEST_NUM_SLAVE_THREADS to 5 (renderd1 = 1, renderd2 = 4 [NUM_THREADS])
+    set(CTEST_NUM_SLAVE_THREADS 5)
+    add_test(
+      NAME renderd_num_threads_${STORAGE_BACKEND}
+      COMMAND ${BASH} -c "
+        if ! ${GREP_EXECUTABLE} -q \"config renderd: num_threads=${PROCESSOR_COUNT}\" \"${RENDERD0_LOG}\"; then
+          ${GREP_EXECUTABLE} \"config renderd: num_threads=\" \"${RENDERD0_LOG}\"
+          exit 1;
+        fi
+        if ! ${GREP_EXECUTABLE} -q \"config renderd: num_slave_threads=${CTEST_NUM_SLAVE_THREADS}\" \"${RENDERD0_LOG}\"; then
+          ${GREP_EXECUTABLE} \"config renderd: num_slave_threads=\" \"${RENDERD0_LOG}\"
+          exit 1;
+        fi
+      "
+      WORKING_DIRECTORY tests
+    )
+    set_tests_properties(renderd_num_threads_${STORAGE_BACKEND} PROPERTIES
+      FIXTURES_REQUIRED services_started_${STORAGE_BACKEND}
+      REQUIRED_FILES ${RENDERD0_LOG}
+    )
+  endif()
   add_test(
     NAME stop_services_${STORAGE_BACKEND}
     COMMAND ${BASH} -c "

tests/renderd.conf.in

@@ -42,11 +42,13 @@ XML=@PROJECT_SOURCE_DIR@/utils/example-map/mapnik.xml
 [renderd1]
 iphostname=@RENDERD1_HOST@
 ipport=@RENDERD1_PORT@
+num_threads=1
 pid_file=@RENDERD1_PID@
 stats_file=@TEST_RUN_DIR@/renderd1_@STORAGE_BACKEND@.stats
 tile_dir=@TILE_DIR@
 
 [renderd0]
+num_threads=-1
 pid_file=@RENDERD0_PID@
 socketname=@RENDERD0_SOCKET@
 stats_file=@TEST_RUN_DIR@/renderd0_@STORAGE_BACKEND@.stats