Skip to content

Commit 6ac9d14

Browse files
jakeaturnerjakeaturner
committed
feat(Collections): map region collections
1 parent f618512 commit 6ac9d14

File tree

12 files changed

+474
-198
lines changed

12 files changed

+474
-198
lines changed

‎admin/app/controllers/maps_controller.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { MapService } from '#services/map_service'
22
import {
3+
downloadCollectionValidator,
34
filenameParamValidator,
45
remoteDownloadValidator,
56
remoteDownloadValidatorOptional,
@@ -36,13 +37,32 @@ export default class MapsController {
3637
}
3738
}
3839

40+
async downloadCollection({ request }: HttpContext) {
41+
const payload = await request.validateUsing(downloadCollectionValidator)
42+
const resources = await this.mapService.downloadCollection(payload.slug)
43+
return {
44+
message: 'Collection download started successfully',
45+
slug: payload.slug,
46+
resources,
47+
}
48+
}
49+
3950
// For providing a "preflight" check in the UI before actually starting a background download
4051
async downloadRemotePreflight({ request }: HttpContext) {
4152
const payload = await request.validateUsing(remoteDownloadValidator)
4253
const info = await this.mapService.downloadRemotePreflight(payload.url)
4354
return info
4455
}
4556

57+
async fetchLatestCollections({}: HttpContext) {
58+
const success = await this.mapService.fetchLatestCollections()
59+
return { success }
60+
}
61+
62+
async listCuratedCollections({}: HttpContext) {
63+
return await this.mapService.listCuratedCollections()
64+
}
65+
4666
async listRegions({}: HttpContext) {
4767
return await this.mapService.listRegions()
4868
}

‎admin/app/controllers/zim_controller.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,12 @@ export default class ZimController {
3636

3737
async downloadCollection({ request }: HttpContext) {
3838
const payload = await request.validateUsing(downloadCollectionValidator)
39-
const resource_count = await this.zimService.downloadCollection(payload.slug)
39+
const resources = await this.zimService.downloadCollection(payload.slug)
4040

4141
return {
4242
message: 'Download started successfully',
4343
slug: payload.slug,
44-
resource_count,
44+
resources,
4545
}
4646
}
4747

‎admin/app/jobs/run_download_job.ts‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { doResumableDownload } from '../utils/downloads.js'
55
import { createHash } from 'crypto'
66
import { DockerService } from '#services/docker_service'
77
import { ZimService } from '#services/zim_service'
8+
import { MapService } from '#services/map_service'
89

910
export class RunDownloadJob {
1011
static get queue() {
@@ -23,9 +24,9 @@ export class RunDownloadJob {
2324
const { url, filepath, timeout, allowedMimeTypes, forceNew, filetype } =
2425
job.data as RunDownloadJobParams
2526

26-
// console.log("Simulating delay for job for URL:", url)
27-
// await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
28-
// console.log("Starting download for URL:", url)
27+
// console.log("Simulating delay for job for URL:", url)
28+
// await new Promise((resolve) => setTimeout(resolve, 30000)) // Simulate initial delay
29+
// console.log("Starting download for URL:", url)
2930

3031
// // simulate progress updates for demonstration
3132
// for (let progress = 0; progress <= 100; progress += 10) {
@@ -45,17 +46,20 @@ export class RunDownloadJob {
4546
job.updateProgress(Math.floor(progressPercent))
4647
},
4748
async onComplete(url) {
48-
if (filetype === 'zim') {
49-
try {
49+
try {
50+
if (filetype === 'zim') {
5051
const dockerService = new DockerService()
5152
const zimService = new ZimService(dockerService)
5253
await zimService.downloadRemoteSuccessCallback([url], true)
53-
} catch (error) {
54-
console.error(
55-
`[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
56-
error
57-
)
54+
} else if (filetype === 'map') {
55+
const mapsService = new MapService()
56+
await mapsService.downloadRemoteSuccessCallback([url], false)
5857
}
58+
} catch (error) {
59+
console.error(
60+
`[RunDownloadJob] Error in ZIM download success callback for URL ${url}:`,
61+
error
62+
)
5963
}
6064
job.updateProgress(100)
6165
},

‎admin/app/services/map_service.ts‎

Lines changed: 117 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import { BaseStylesFile, MapLayer } from '../../types/maps.js'
2-
import { FileEntry } from '../../types/files.js'
2+
import {
3+
DownloadCollectionOperation,
4+
DownloadRemoteSuccessCallback,
5+
FileEntry,
6+
} from '../../types/files.js'
37
import { doResumableDownloadWithRetry } from '../utils/downloads.js'
48
import { extract } from 'tar'
59
import env from '#start/env'
@@ -16,18 +20,31 @@ import urlJoin from 'url-join'
1620
import axios from 'axios'
1721
import { RunDownloadJob } from '#jobs/run_download_job'
1822
import logger from '@adonisjs/core/services/logger'
23+
import { CuratedCollectionsFile, CuratedCollectionWithStatus } from '../../types/downloads.js'
24+
import CuratedCollection from '#models/curated_collection'
25+
import vine from '@vinejs/vine'
26+
import { curatedCollectionsFileSchema } from '#validators/curated_collections'
27+
import CuratedCollectionResource from '#models/curated_collection_resource'
1928

2029
const BASE_ASSETS_MIME_TYPES = [
2130
'application/gzip',
2231
'application/x-gzip',
2332
'application/octet-stream',
2433
]
2534

35+
const COLLECTIONS_URL =
36+
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/maps.json'
37+
2638
const PMTILES_ATTRIBUTION =
2739
'<a href="https://github.com/protomaps/basemaps">Protomaps</a> © <a href="https://openstreetmap.org">OpenStreetMap</a>'
2840
const PMTILES_MIME_TYPES = ['application/vnd.pmtiles', 'application/octet-stream']
2941

30-
export class MapService {
42+
interface IMapService {
43+
downloadCollection: DownloadCollectionOperation
44+
downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback
45+
}
46+
47+
export class MapService implements IMapService {
3148
private readonly mapStoragePath = '/storage/maps'
3249
private readonly baseStylesFile = 'nomad-base-styles.json'
3350
private readonly basemapsAssetsDir = 'basemaps-assets'
@@ -80,6 +97,62 @@ export class MapService {
8097
return true
8198
}
8299

100+
async downloadCollection(slug: string) {
101+
const collection = await CuratedCollection.query()
102+
.where('slug', slug)
103+
.andWhere('type', 'map')
104+
.first()
105+
if (!collection) {
106+
return null
107+
}
108+
109+
const resources = await collection.related('resources').query().where('downloaded', false)
110+
if (resources.length === 0) {
111+
return null
112+
}
113+
114+
const downloadUrls = resources.map((res) => res.url)
115+
const downloadFilenames: string[] = []
116+
117+
for (const url of downloadUrls) {
118+
const existing = await RunDownloadJob.getByUrl(url)
119+
if (existing) {
120+
logger.warn(`[MapService] Download already in progress for URL ${url}, skipping.`)
121+
continue
122+
}
123+
124+
// Extract the filename from the URL
125+
const filename = url.split('/').pop()
126+
if (!filename) {
127+
logger.warn(`[MapService] Could not determine filename from URL ${url}, skipping.`)
128+
continue
129+
}
130+
131+
downloadFilenames.push(filename)
132+
const filepath = join(process.cwd(), this.mapStoragePath, 'pmtiles', filename)
133+
134+
await RunDownloadJob.dispatch({
135+
url,
136+
filepath,
137+
timeout: 30000,
138+
allowedMimeTypes: PMTILES_MIME_TYPES,
139+
forceNew: true,
140+
filetype: 'map',
141+
})
142+
}
143+
144+
return downloadFilenames.length > 0 ? downloadFilenames : null
145+
}
146+
147+
async downloadRemoteSuccessCallback(urls: string[], _: boolean) {
148+
const resources = await CuratedCollectionResource.query().whereIn('url', urls)
149+
for (const resource of resources) {
150+
resource.downloaded = true
151+
await resource.save()
152+
logger.info(`[MapService] Marked resource as downloaded: ${resource.url}`)
153+
}
154+
}
155+
83156
async downloadRemote(url: string): Promise<{ filename: string; jobId?: string }> {
84157
const parsed = new URL(url)
85158
if (!parsed.pathname.endsWith('.pmtiles')) {
@@ -105,7 +178,7 @@ export class MapService {
105178
timeout: 30000,
106179
allowedMimeTypes: PMTILES_MIME_TYPES,
107180
forceNew: true,
108-
filetype: 'pmtiles',
181+
filetype: 'map',
109182
})
110183

111184
if (!result.job) {
@@ -193,6 +266,47 @@ export class MapService {
193266
return !!baseStyleItem && !!basemapsAssetsItem
194267
}
195268

269+
async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
270+
const collections = await CuratedCollection.query().where('type', 'map').preload('resources')
271+
return collections.map((collection) => ({
272+
...(collection.serialize() as CuratedCollection),
273+
all_downloaded: collection.resources.every((res) => res.downloaded),
274+
}))
275+
}
276+
277+
async fetchLatestCollections(): Promise<boolean> {
278+
try {
279+
const response = await axios.get<CuratedCollectionsFile>(COLLECTIONS_URL)
280+
281+
const validated = await vine.validate({
282+
schema: curatedCollectionsFileSchema,
283+
data: response.data,
284+
})
285+
286+
for (const collection of validated.collections) {
287+
const collectionResult = await CuratedCollection.updateOrCreate(
288+
{ slug: collection.slug },
289+
{
290+
...collection,
291+
type: 'map',
292+
}
293+
)
294+
logger.info(`[MapService] Upserted curated collection: ${collection.slug}`)
295+
296+
await collectionResult.related('resources').createMany(collection.resources)
297+
logger.info(
298+
`[MapService] Upserted ${collection.resources.length} resources for collection: ${collection.slug}`
299+
)
300+
}
301+
302+
return true
303+
} catch (error) {
304+
console.error(error)
305+
logger.error(`[MapService] Failed to download latest Kiwix collections:`, error)
306+
return false
307+
}
308+
}
309+
196310
private async listMapStorageItems(): Promise<FileEntry[]> {
197311
await ensureDirectoryExists(this.baseDirPath)
198312
return await listDirectoryContents(this.baseDirPath)

‎admin/app/services/zim_service.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,20 @@ import { curatedCollectionsFileSchema } from '#validators/curated_collections'
2323
import CuratedCollection from '#models/curated_collection'
2424
import CuratedCollectionResource from '#models/curated_collection_resource'
2525
import { RunDownloadJob } from '#jobs/run_download_job'
26+
import { DownloadCollectionOperation, DownloadRemoteSuccessCallback } from '../../types/files.js'
2627

2728
const ZIM_MIME_TYPES = ['application/x-zim', 'application/x-openzim', 'application/octet-stream']
2829
const COLLECTIONS_URL =
2930
'https://github.com/Crosstalk-Solutions/project-nomad/raw/refs/heads/master/collections/kiwix.json'
3031

32+
33+
interface IZimService {
34+
downloadCollection: DownloadCollectionOperation
35+
downloadRemoteSuccessCallback: DownloadRemoteSuccessCallback
36+
}
37+
3138
@inject()
32-
export class ZimService {
39+
export class ZimService implements IZimService {
3340
constructor(private dockerService: DockerService) {}
3441

3542
async list() {
@@ -176,8 +183,8 @@ export class ZimService {
176183
}
177184
}
178185

179-
async downloadCollection(slug: string): Promise<string[] | null> {
180-
const collection = await CuratedCollection.find(slug)
186+
async downloadCollection(slug: string) {
187+
const collection = await CuratedCollection.query().where('slug', slug).andWhere('type', 'zim').first()
181188
if (!collection) {
182189
return null
183190
}
@@ -218,7 +225,7 @@ export class ZimService {
218225
}
219226

220227
return downloadFilenames.length > 0 ? downloadFilenames : null
221-
}
228+
}
222229

223230
async downloadRemoteSuccessCallback(urls: string[], restart = true) {
224231
// Restart KIWIX container to pick up new ZIM file
@@ -239,7 +246,7 @@ export class ZimService {
239246
}
240247

241248
async listCuratedCollections(): Promise<CuratedCollectionWithStatus[]> {
242-
const collections = await CuratedCollection.query().preload('resources')
249+
const collections = await CuratedCollection.query().where('type', 'zim').preload('resources')
243250
return collections.map((collection) => ({
244251
...(collection.serialize() as CuratedCollection),
245252
all_downloaded: collection.resources.every((res) => res.downloaded),

0 commit comments

Comments
 (0)