Skip to content

Commit 2099750

Browse files
jakeaturnerjakeaturner
committed
fix(OSM): osm installation
1 parent 5ee949b commit 2099750

File tree

6 files changed

+138
-102
lines changed

6 files changed

+138
-102
lines changed

‎admin/app/controllers/home_controller.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ export default class HomeController {
1515

1616
async home({ inertia }: HttpContext) {
1717
const services = await this.systemService.getServices({ installedOnly: true });
18-
console.log(services)
1918
return inertia.render('home', {
2019
system: {
2120
services

‎admin/app/services/docker_service.ts‎

Lines changed: 87 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@ import Docker from "dockerode";
33
import drive from '@adonisjs/drive/services/main'
44
import axios from 'axios';
55
import logger from '@adonisjs/core/services/logger'
6-
import transmit from '@adonisjs/transmit/services/main'
76
import { inject } from "@adonisjs/core";
87
import { ServiceStatus } from "../../types/services.js";
8+
import transmit from "@adonisjs/transmit/services/main";
9+
import { Readable } from "stream";
10+
import { chmodRecursive, chownRecursive } from "../../util/files.js";
11+
import fs from 'fs'
912

1013
@inject()
1114
export class DockerService {
1215
private docker: Docker;
16+
public static NOMAD_STORAGE_DIR = '/opt/project-nomad/storage';
1317
public static KIWIX_SERVICE_NAME = 'nomad_kiwix_serve';
1418
public static OPENSTREETMAP_SERVICE_NAME = 'nomad_openstreetmap';
1519
public static OPENSTREETMAP_IMPORT_SERVICE_NAME = 'nomad_openstreetmap_import';
@@ -71,7 +75,7 @@ export class DockerService {
7175
message: `Invalid action: ${action}. Use 'start', 'stop', or 'restart'.`,
7276
}
7377
} catch (error) {
74-
console.error(`Error starting service ${serviceName}: ${error.message}`);
78+
logger.error(`Error starting service ${serviceName}: ${error.message}`);
7579
return {
7680
success: false,
7781
message: `Failed to start service ${serviceName}: ${error.message}`,
@@ -170,7 +174,7 @@ export class DockerService {
170174
*/
171175
async _createContainer(service: Service & { dependencies?: Service[] }, containerConfig: any): Promise<void> {
172176
try {
173-
this._broadcastAndLog(service.service_name, 'initializing', '');
177+
this._broadcast(service.service_name, 'initializing', '');
174178

175179
let dependencies = [];
176180
if (service.depends_on) {
@@ -182,55 +186,52 @@ export class DockerService {
182186

183187
// First, check if the service has any dependencies that need to be installed first
184188
if (dependencies && dependencies.length > 0) {
185-
this._broadcastAndLog(service.service_name, 'checking-dependencies', `Checking dependencies for service ${service.service_name}...`);
189+
this._broadcast(service.service_name, 'checking-dependencies', `Checking dependencies for service ${service.service_name}...`);
186190
for (const dependency of dependencies) {
187191
if (!dependency.installed) {
188-
this._broadcastAndLog(service.service_name, 'dependency-not-installed', `Dependency service ${dependency.service_name} is not installed. Installing it first...`);
192+
this._broadcast(service.service_name, 'dependency-not-installed', `Dependency service ${dependency.service_name} is not installed. Installing it first...`);
189193
await this._createContainer(dependency, this._parseContainerConfig(dependency.container_config));
190194
} else {
191-
this._broadcastAndLog(service.service_name, 'dependency-installed', `Dependency service ${dependency.service_name} is already installed.`);
195+
this._broadcast(service.service_name, 'dependency-installed', `Dependency service ${dependency.service_name} is already installed.`);
192196
}
193197
}
194198
}
195199

196200
// Start pulling the Docker image and wait for it to complete
197201
const pullStream = await this.docker.pull(service.container_image);
198-
this._broadcastAndLog(service.service_name, 'pulling', `Pulling Docker image ${service.container_image}...`);
202+
this._broadcast(service.service_name, 'pulling', `Pulling Docker image ${service.container_image}...`);
199203
await new Promise(res => this.docker.modem.followProgress(pullStream, res));
200204

201-
this._broadcastAndLog(service.service_name, 'creating', `Creating Docker container for service ${service.service_name}...`);
205+
if (service.service_name === DockerService.KIWIX_SERVICE_NAME) {
206+
await this._runPreinstallActions__KiwixServe();
207+
this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`);
208+
} else if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) {
209+
await this._runPreinstallActions__OpenStreetMap(service.container_image, containerConfig);
210+
this._broadcast(service.service_name, 'preinstall-complete', `Pre-install actions for OpenStreetMap completed successfully.`);
211+
}
212+
213+
this._broadcast(service.service_name, 'creating', `Creating Docker container for service ${service.service_name}...`);
202214
const container = await this.docker.createContainer({
203215
Image: service.container_image,
204216
name: service.service_name,
217+
...(containerConfig?.User && { User: containerConfig.User }),
205218
...(containerConfig?.HostConfig && { HostConfig: containerConfig.HostConfig }),
206219
...(containerConfig?.WorkingDir && { WorkingDir: containerConfig.WorkingDir }),
207220
...(containerConfig?.ExposedPorts && { ExposedPorts: containerConfig.ExposedPorts }),
208221
...(service.container_command ? { Cmd: service.container_command.split(' ') } : {}),
209222
...(service.service_name === 'open-webui' ? { Env: ['WEBUI_AUTH=False', 'PORT=3000', 'OLLAMA_BASE_URL=http://127.0.0.1:11434'] } : {}), // Special case for Open WebUI
210223
});
211224

212-
if (service.service_name === DockerService.KIWIX_SERVICE_NAME) {
213-
await this._runPreinstallActions__KiwixServe();
214-
this._broadcastAndLog(service.service_name, 'preinstall-complete', `Pre-install actions for Kiwix Serve completed successfully.`);
215-
} else if (service.service_name === DockerService.OPENSTREETMAP_SERVICE_NAME) {
216-
await this._runPreinstallActions__OpenStreetMap(service.container_image, containerConfig);
217-
this._broadcastAndLog(service.service_name, 'preinstall-complete', `Pre-install actions for OpenStreetMap completed successfully.`);
218-
}
219-
220-
console.log("GOT HERE")
221-
222-
this._broadcastAndLog(service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...`);
225+
this._broadcast(service.service_name, 'starting', `Starting Docker container for service ${service.service_name}...`);
223226
await container.start();
224227

225-
console.log("GOT HERE 2")
226-
this._broadcastAndLog(service.service_name, 'finalizing', `Finalizing installation of service ${service.service_name}...`);
228+
this._broadcast(service.service_name, 'finalizing', `Finalizing installation of service ${service.service_name}...`);
227229
service.installed = true;
228230
await service.save();
229231

230-
console.log("GOT HERE 3")
231-
this._broadcastAndLog(service.service_name, 'completed', `Service ${service.service_name} installation completed successfully.`);
232+
this._broadcast(service.service_name, 'completed', `Service ${service.service_name} installation completed successfully.`);
232233
} catch (error) {
233-
this._broadcastAndLog(service.service_name, 'error', `Error installing service ${service.service_name}: ${error.message}`);
234+
this._broadcast(service.service_name, 'error', `Error installing service ${service.service_name}: ${error.message}`);
234235
throw new Error(`Failed to install service ${service.service_name}: ${error.message}`);
235236
}
236237
}
@@ -240,7 +241,7 @@ export class DockerService {
240241
const containers = await this.docker.listContainers({ all: true });
241242
return containers.some(container => container.Names.includes(`/${serviceName}`));
242243
} catch (error) {
243-
console.error(`Error checking if service container exists: ${error.message}`);
244+
logger.error(`Error checking if service container exists: ${error.message}`);
244245
return false;
245246
}
246247
}
@@ -259,7 +260,7 @@ export class DockerService {
259260

260261
return { success: true, message: `Service ${serviceName} container removed successfully` };
261262
} catch (error) {
262-
console.error(`Error removing service container: ${error.message}`);
263+
logger.error(`Error removing service container: ${error.message}`);
263264
return { success: false, message: `Failed to remove service ${serviceName} container: ${error.message}` };
264265
}
265266
}
@@ -272,8 +273,8 @@ export class DockerService {
272273
const WIKIPEDIA_ZIM_URL = "https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_mini_2025-06.zim"
273274
const PATH = '/zim/wikipedia_en_100_mini_2025-06.zim';
274275

275-
this._broadcastAndLog(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Running pre-install actions for Kiwix Serve...`);
276-
this._broadcastAndLog(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`);
276+
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Running pre-install actions for Kiwix Serve...`);
277+
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloading Wikipedia ZIM file from ${WIKIPEDIA_ZIM_URL}. This may take some time...`);
277278
const response = await axios.get(WIKIPEDIA_ZIM_URL, {
278279
responseType: 'stream',
279280
});
@@ -287,89 +288,84 @@ export class DockerService {
287288
const disk = drive.use('fs');
288289
await disk.putStream(PATH, stream);
289290

290-
this._broadcastAndLog(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloaded Wikipedia ZIM file to ${PATH}`);
291+
this._broadcast(DockerService.KIWIX_SERVICE_NAME, 'preinstall', `Downloaded Wikipedia ZIM file to ${PATH}`);
291292
}
292293

293294
/**
294295
* Largely follows the install instructions here: https://github.com/Overv/openstreetmap-tile-server/blob/master/README.md
295296
*/
296297
private async _runPreinstallActions__OpenStreetMap(image: string, containerConfig: any): Promise<void> {
297-
const FILE_NAME = 'us-pacific-latest.osm.pbf';
298-
const OSM_PBF_URL = `https://download.geofabrik.de/north-america/${FILE_NAME}`; // Download a small subregion for initial import
299-
const PATH = `/osm/${FILE_NAME}`;
300-
const IMPORT_BIND = `${PATH}:/data/region.osm.pbf`;
298+
const OSM_PBF_URL = 'https://download.geofabrik.de/north-america/us-pacific-latest.osm.pbf'; // Download a small subregion for initial import
299+
const IMPORT_FILE = 'region.osm.pbf';
300+
const PATH = `${DockerService.NOMAD_STORAGE_DIR}/osm/${IMPORT_FILE}`;
301+
const IMPORT_BIND = `${PATH}:/data/${IMPORT_FILE}:rw`;
302+
const LOG_PATH = `${DockerService.NOMAD_STORAGE_DIR}/logs/${DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME}.log`;
301303
const disk = drive.use('fs');
302304

303-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Running pre-install actions for OpenStreetMap Tile Server...`);
305+
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Running pre-install actions for OpenStreetMap Tile Server...`);
304306

305-
const fileExists = await disk.exists(PATH);
306-
if (!fileExists) {
307-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Downloading OpenStreetMap PBF file from ${OSM_PBF_URL}. This may take some time...`);
308-
const response = await axios.get(OSM_PBF_URL, {
309-
responseType: 'stream',
310-
});
307+
// Ensure osm directory has proper perms for OSM container to write cached files to
308+
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', 'Ensuring OSM directory permissions are set correctly...');
311309

312-
const stream = response.data;
313-
stream.on('error', (error: Error) => {
314-
logger.error(`Error downloading OpenStreetMap PBF file: ${error.message}`);
315-
throw error;
316-
});
310+
// Ensure directories exist
311+
await fs.promises.mkdir(`${DockerService.NOMAD_STORAGE_DIR}/osm/db`, { recursive: true });
312+
await fs.promises.mkdir(`${DockerService.NOMAD_STORAGE_DIR}/osm/tiles`, { recursive: true });
313+
314+
await chmodRecursive(`${DockerService.NOMAD_STORAGE_DIR}/osm/db`, 0o755, 0o755); // Must be able to read directories and read/write files inside
315+
await chownRecursive(`${DockerService.NOMAD_STORAGE_DIR}/osm/db`, 1000, 1000);
317316

318-
await disk.putStream(PATH, stream);
319-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Downloaded OpenStreetMap PBF file to ${PATH}`);
320-
} else {
321-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `OpenStreetMap PBF file already exists at ${PATH}. Skipping download.`);
317+
await chmodRecursive(`${DockerService.NOMAD_STORAGE_DIR}/osm/tiles`, 0o755, 0o755); // Must be able to read directories and read/write files inside
318+
await chownRecursive(`${DockerService.NOMAD_STORAGE_DIR}/osm/tiles`, 1000, 1000);
319+
320+
321+
// If the initial import file already exists, delete it so we can ensure it is a good download
322+
const fileExists = await disk.exists(PATH);
323+
if (fileExists) {
324+
await disk.delete(PATH);
322325
}
323326

327+
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'preinstall', `Downloading OpenStreetMap PBF file from ${OSM_PBF_URL}. This may take some time...`);
328+
const response = await axios.get(OSM_PBF_URL, {
329+
responseType: 'stream',
330+
});
331+
await disk.putStream(PATH, response.data);
332+
324333
// Do initial import of OSM data into the tile server DB
325334
// We need to add the initial osm.pbf file as another volume bind so we can import it
326335
const configWithImportBind = containerConfig.HostConfig || {};
327-
const bindsArray: string[] = []
328-
if (Array.isArray(configWithImportBind.Binds)) {
329-
bindsArray.push(...configWithImportBind.Binds, IMPORT_BIND);
330-
} else {
331-
bindsArray.push(IMPORT_BIND);
332-
}
333-
configWithImportBind.Binds = bindsArray;
334-
335-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'importing', `Processing initial import of OSM data. This may take some time...`);
336-
337-
338-
const result = await new Promise((resolve, reject) => {
339-
this.docker.run(image,
340-
['import'],
341-
process.stdout,
342-
configWithImportBind,
343-
{},
344-
{},
345-
// @ts-ignore
346-
(err: any, data: any, container: Docker.Container) => {
347-
if (err) {
348-
logger.error(`Error running initial import for OpenStreetMap Tile Server: ${err.message}`);
349-
return reject(err);
350-
}
351-
resolve(data);
352-
});
353-
}).catch((error) => {
354-
logger.error(`Error during OpenStreetMap data import: ${error.message}`);
355-
return null;
336+
Object.assign(configWithImportBind, {
337+
RestartPolicy: { Name: 'no' },
338+
Binds: [...(containerConfig.HostConfig?.Binds || []), IMPORT_BIND],
356339
});
357340

358-
logger.log('debug', `OpenStreetMap data import result: ${JSON.stringify(result)}`);
359-
360-
const [output, container] = result ? result as [any, any] : [null, null];
361-
if (output?.StatusCode === 0) {
362-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'imported', `OpenStreetMap data imported successfully.`);
363-
await container.remove();
364-
} else {
365-
const errorMessage = `Failed to import OpenStreetMap data. Status code: ${output?.StatusCode}. Output: ${output?.Output || 'No output'}`;
366-
this._broadcastAndLog(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'error', errorMessage);
367-
logger.error(errorMessage);
368-
throw new Error(errorMessage);
369-
}
341+
this._broadcast(DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME, 'importing', `Processing initial import of OSM data. This may take some time...`);
342+
343+
const container = await this.docker.createContainer({
344+
Image: image,
345+
name: DockerService.OPENSTREETMAP_IMPORT_SERVICE_NAME,
346+
Cmd: ['import'],
347+
HostConfig: configWithImportBind,
348+
});
349+
350+
await container.start();
351+
352+
const logStream = await container.logs({
353+
stdout: true,
354+
stderr: true,
355+
follow: true,
356+
timestamps: true
357+
})
358+
359+
const readableLogStream: Readable = Readable.from(logStream);
360+
await disk.putStream(LOG_PATH, readableLogStream);
361+
362+
const data = await container.wait();
363+
logger.debug(`OpenStreetMap data import result: ${JSON.stringify(data)}`);
364+
365+
await container.remove();
370366
}
371367

372-
private _broadcastAndLog(service: string, status: string, message: string) {
368+
private _broadcast(service: string, status: string, message: string) {
373369
transmit.broadcast('service-installation', {
374370
service_name: service,
375371
timestamp: new Date().toISOString(),

‎admin/database/seeders/service_seeder.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { BaseSeeder } from '@adonisjs/lucid/seeders'
44
import { ModelAttributes } from '@adonisjs/lucid/types/model'
55

66
export default class ServiceSeeder extends BaseSeeder {
7-
private static NOMAD_STORAGE_DIR = '/opt/project-nomad/storage'
87
private static DEFAULT_SERVICES: Omit<ModelAttributes<Service>, 'created_at' | 'updated_at' | 'metadata' | 'id'>[] = [
98
{
109
service_name: DockerService.KIWIX_SERVICE_NAME,
@@ -13,7 +12,7 @@ export default class ServiceSeeder extends BaseSeeder {
1312
container_config: JSON.stringify({
1413
HostConfig: {
1514
RestartPolicy: { Name: 'unless-stopped' },
16-
Binds: [`${ServiceSeeder.NOMAD_STORAGE_DIR}/zim:/data`],
15+
Binds: [`${DockerService.NOMAD_STORAGE_DIR}/zim:/data`],
1716
PortBindings: { '8080/tcp': [{ HostPort: '8090' }] }
1817
},
1918
ExposedPorts: { '8080/tcp': {} }
@@ -31,8 +30,8 @@ export default class ServiceSeeder extends BaseSeeder {
3130
HostConfig: {
3231
RestartPolicy: { Name: 'unless-stopped' },
3332
Binds: [
34-
`${ServiceSeeder.NOMAD_STORAGE_DIR}/osm/db:/data/database`,
35-
`${ServiceSeeder.NOMAD_STORAGE_DIR}/osm/tiles:/data/tiles`
33+
`${DockerService.NOMAD_STORAGE_DIR}/osm/db:/data/database:rw`,
34+
`${DockerService.NOMAD_STORAGE_DIR}/osm/tiles:/data/tiles:rw`
3635
],
3736
PortBindings: { '80/tcp': [{ HostPort: '9000' }] }
3837
}
@@ -49,7 +48,7 @@ export default class ServiceSeeder extends BaseSeeder {
4948
container_config: JSON.stringify({
5049
HostConfig: {
5150
RestartPolicy: { Name: 'unless-stopped' },
52-
Binds: [`${ServiceSeeder.NOMAD_STORAGE_DIR}/ollama:/root/.ollama`],
51+
Binds: [`${DockerService.NOMAD_STORAGE_DIR}/ollama:/root/.ollama`],
5352
PortBindings: { '11434/tcp': [{ HostPort: '11434' }] }
5453
},
5554
ExposedPorts: { '11434/tcp': {} }
@@ -67,7 +66,7 @@ export default class ServiceSeeder extends BaseSeeder {
6766
HostConfig: {
6867
RestartPolicy: { Name: 'unless-stopped' },
6968
NetworkMode: 'host',
70-
Binds: [`${ServiceSeeder.NOMAD_STORAGE_DIR}/open-webui:/app/backend/data`]
69+
Binds: [`${DockerService.NOMAD_STORAGE_DIR}/open-webui:/app/backend/data`]
7170
}
7271
}),
7372
ui_location: '3000',

‎admin/inertia/pages/home.tsx‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,7 @@ export default function Home(props: {
3737
}
3838
}) {
3939
const items = []
40-
41-
console.log(props.system.services)
42-
props.system.services.map((service) => {
40+
props.system.services.map((service) => {
4341
items.push({
4442
label: service.service_name,
4543
to: getServiceLink(service.ui_location),

‎admin/inertia/pages/settings/apps.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ export default function SettingsPage(props: { system: { services: ServiceSlim[]
2525

2626
useEffect(() => {
2727
const unsubscribe = subscribe('service-installation', (data: any) => {
28-
console.log('Received service installation message:', data)
2928
setInstallActivity((prev) => [
3029
...prev,
3130
{

0 commit comments

Comments
 (0)