Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion src/api/FaableApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ export type FaableVolumeOperation = {
id: string
volume_id: string
app_id: string
kind: 'backup' | 'restore' | 'export' | 'import' | 'put'
kind: 'backup' | 'restore' | 'export' | 'import' | 'put' | 'list'
reason: 'manual' | 'scheduled' | 'pre_restore' | 'pre_import'
state: 'awaiting_upload' | 'queued' | 'running' | 'completed' | 'failed'
backup_id: string | null
Expand All @@ -354,13 +354,24 @@ export type FaableVolumeOperation = {
started_at: string | null
finished_at: string | null
object_deleted_at: string | null
// An export of part of the volume, or the directory a list reads.
path?: string | null
// A completed list: the directory's entries (at most 1000).
entries?: FaableVolumeEntry[] | null
download_url?: string | null
download_expires_at?: string | null
upload_url?: string | null
upload_expires_at?: string | null
createdAt?: string
}

export type FaableVolumeEntry = {
name: string
type: 'file' | 'dir' | 'link' | 'other'
size: number
mtime: string
}

export type FaableVolumeBackups = {
results: FaableVolumeOperation[]
retention: number | null
Expand Down Expand Up @@ -982,6 +993,28 @@ export class FaableApi<T = any> {
)
}

// An export of the whole volume, or of one file or directory of it.
async exportVolume(team: string, volume_id: string, path?: string) {
return data(
this.client.post<FaableVolumeOperation>(
`/volume/${volume_id}/export`,
undefined,
{ params: path ? { path } : {}, headers: projectHeader(team) }
)
)
}

// Queues a read of one directory; its entries come with the completed op.
async listVolumeFiles(team: string, volume_id: string, path?: string) {
return data(
this.client.post<FaableVolumeOperation>(
`/volume/${volume_id}/list-files`,
undefined,
{ params: path ? { path } : {}, headers: projectHeader(team) }
)
)
}

async restoreVolume(team: string, volume_id: string, backup_id: string) {
return data(
this.client.post<FaableVolumeOperation>(
Expand Down
180 changes: 154 additions & 26 deletions src/commands/deploy/volumes/export.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
import axios from 'axios'
import { execFile } from 'node:child_process'
import { createWriteStream } from 'node:fs'
import { rename, rm } from 'node:fs/promises'
import { mkdir, mkdtemp, rename, rm, stat } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { pipeline } from 'node:stream/promises'
import { promisify } from 'node:util'
import { CommandModule } from 'yargs'
import type { FaableApi, FaableVolumeOperation } from '../../../api/FaableApi'
import { CliError } from '../../../lib/errors'
import { json_option, print_json } from '../../../lib/listing'
import { log } from '../../../log'
import { volume_dest } from './operation'
import {
app_option,
format_bytes,
Expand All @@ -16,19 +22,65 @@ import {
wait_operation
} from './shared'

const exec_file = promisify(execFile)

/** Queues an export (of `sub`, or of everything), waits, and saves it to `file`. */
const export_to_file = async (
api: FaableApi,
team: string,
volume_id: string,
sub: string | undefined,
file: string,
quiet: boolean
): Promise<FaableVolumeOperation> => {
const op = await api.exportVolume(team, volume_id, sub)
const done = require_completed(
await wait_operation(api, team, op, {
onState: s => {
if (!quiet && s !== 'completed' && s !== 'failed')
log.info(state_label[s])
}
})
)
// The completed operation read by wait_operation carries a fresh URL.
const url = done.download_url
if (!url) {
throw new CliError(
'not_found',
`The export ${done.id} finished but has no download link; try again`
)
}
const part = `${file}.part`
const res = await axios.get(url, { responseType: 'stream', timeout: 0 })
try {
await pipeline(res.data, createWriteStream(part))
await rename(part, file)
} catch (error) {
await rm(part, { force: true })
throw error
}
return done
}

interface ExportArgs {
app?: string
volume?: string
path?: string
output?: string
json?: boolean
}

export const volumes_export: CommandModule<unknown, ExportArgs> = {
command: 'export',
describe:
'Download the contents of a volume as a .tar.gz (the app keeps running)',
'Download the contents of a volume (or of a path of it) as a .tar.gz; the app keeps running',
builder: yargs =>
volume_option(app_option(json_option(yargs)))
.option('path', {
type: 'string',
description:
'Only this file or directory of the volume (its path inside the archive is the same)'
})
.option('output', {
alias: 'o',
type: 'string',
Expand All @@ -37,41 +89,117 @@ export const volumes_export: CommandModule<unknown, ExportArgs> = {
})
.showHelpOnFail(false) as any,
handler: async args => {
const sub = volume_dest(args.path)
const { api, team, app, volume } = await resolve_volume(
args.app,
args.volume
)
const op = await api.createVolumeOperation(team, volume.id, 'export')
const day = new Date().toISOString().slice(0, 10)
const file = args.output ?? `${app.name}-${volume.name}-${day}.tar.gz`
if (!args.json)
log.info(`📦 Exporting ${app.name}/${volume.name} (${op.id})`)
const done = require_completed(
await wait_operation(api, team, op, {
onState: s => {
if (!args.json && s !== 'completed' && s !== 'failed')
log.info(state_label[s])
}
log.info(
`📦 Exporting ${app.name}/${volume.name}${sub === '.' ? '' : `:${sub}`}`
)
const done = await export_to_file(
api,
team,
volume.id,
sub === '.' ? undefined : sub,
file,
!!args.json
)
if (args.json) return print_json({ ...done, file })
log.info(`✅ Saved ${file} (${format_bytes(done.size_bytes)}).`)
}
}

interface GetArgs {
path: string
app?: string
volume?: string
output?: string
json?: boolean
}

export const volumes_get: CommandModule<unknown, GetArgs> = {
command: 'get <path>',
describe:
'Copy a file or directory of a volume here, like cp: it lands as <output>/<its name>. Existing local files are never overwritten',
builder: yargs =>
volume_option(app_option(json_option(yargs)))
.positional('path', {
type: 'string',
demandOption: true,
description: 'A file or directory of the volume, relative to its root'
})
.option('output', {
alias: 'o',
type: 'string',
default: '.',
description: 'The local directory it goes into'
})
.example(
'$0 deploy volumes get wp-content/mu-plugins/my-plugin -a my-blog',
'A mu-plugin of a WordPress app into ./my-plugin'
)
.showHelpOnFail(false) as any,
handler: async args => {
const sub = volume_dest(args.path)
if (sub === '.') {
throw new CliError(
'usage',
'get copies a file or directory; the whole volume is faable deploy volumes export'
)
}
const { api, team, app, volume } = await resolve_volume(
args.app,
args.volume
)
// The completed operation read by wait_operation carries a fresh URL.
const url = done.download_url
if (!url) {
const out = args.output ?? '.'
// Like cp, but never over what is here: GNU tar -k fails on an existing
// file, macOS tar skips it in silence; say it before downloading.
const local_target = path.join(out, path.posix.basename(sub))
if (await stat(local_target).catch(() => null)) {
throw new CliError(
'not_found',
`The export ${done.id} finished but has no download link; try faable deploy volumes export again`
'usage',
`${local_target} already exists: nothing is overwritten (move it, or --output another directory)`
)
}
const day = new Date().toISOString().slice(0, 10)
const file = args.output ?? `${app.name}-${volume.name}-${day}.tar.gz`
const part = `${file}.part`
const res = await axios.get(url, { responseType: 'stream', timeout: 0 })
const work = await mkdtemp(path.join(tmpdir(), 'faable-volume-get-'))
try {
await pipeline(res.data, createWriteStream(part))
await rename(part, file)
} catch (error) {
await rm(part, { force: true })
throw error
if (!args.json)
log.info(`📥 Copying ${app.name}/${volume.name}:${sub} into ${out}`)
const archive = path.join(work, 'get.tar.gz')
const done = await export_to_file(
api,
team,
volume.id,
sub,
archive,
!!args.json
)
await mkdir(out, { recursive: true })
// The archive holds the path as it is in the volume: strip its parents
// so it lands as <out>/<name>. -k: an existing local file is an error,
// never overwritten.
const strip = sub.split('/').length - 1
await exec_file('tar', [
'-xzkf',
archive,
'-C',
out,
...(strip ? [`--strip-components=${strip}`] : [])
]).catch((error: { stderr?: string }) => {
throw new CliError(
'usage',
`Could not unpack into ${out}: ${(error.stderr ?? String(error)).trim().split('\n')[0]}`
)
})
const local = local_target
if (args.json) return print_json({ ...done, local })
log.info(`✅ ${local} (${format_bytes(done.size_bytes)} compressed).`)
} finally {
await rm(work, { recursive: true, force: true })
}
if (args.json) return print_json({ ...done, file })
log.info(`✅ Saved ${file} (${format_bytes(done.size_bytes)}).`)
}
}
9 changes: 6 additions & 3 deletions src/commands/deploy/volumes/index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { CommandModule } from 'yargs'
import { volumes_backup, volumes_backups } from './backups'
import { volumes_export } from './export'
import { volumes_export, volumes_get } from './export'
import { volumes_import, volumes_put } from './import'
import { volumes_list } from './list'
import { volumes_ls } from './ls'
import { volumes_restore } from './restore'

// The volumes of an app (api `/volume`, `/volume-operation`): today the ones
Expand All @@ -11,11 +12,13 @@ import { volumes_restore } from './restore'
// describe below. Arch: deploy/volumenes-cli-comparativa.md (backlog §460).
export const volumes: CommandModule = {
command: 'volumes <command>',
// 'Manage the volumes of an app: usage, backups, restore, export, import and put'
// 'Manage the volumes of an app: usage, files (ls, get, put), backups, restore, export and import'
describe: false,
builder: yargs =>
yargs
.command(volumes_list)
.command(volumes_ls)
.command(volumes_get)
.command(volumes_backups)
.command(volumes_backup)
.command(volumes_restore)
Expand All @@ -24,7 +27,7 @@ export const volumes: CommandModule = {
.command(volumes_put)
.demandCommand(
1,
'Specify a volumes command: list, backups, backup, restore, export, import or put'
'Specify a volumes command: list, ls, get, backups, backup, restore, export, import or put'
),
handler: () => {
// Unreachable: demandCommand(1) either routes to a subcommand or fails
Expand Down
59 changes: 59 additions & 0 deletions src/commands/deploy/volumes/ls.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { CommandModule } from 'yargs'
import { json_option, print, print_json } from '../../../lib/listing'
import { log } from '../../../log'
import { format_entry, volume_dest } from './operation'
import {
app_option,
require_completed,
resolve_volume,
volume_option,
wait_operation
} from './shared'

interface LsArgs {
path?: string
app?: string
volume?: string
json?: boolean
}

export const volumes_ls: CommandModule<unknown, LsArgs> = {
command: 'ls [path]',
describe:
'List a directory of a volume (default its root), with the app serving',
builder: yargs =>
volume_option(app_option(json_option(yargs)))
.positional('path', {
type: 'string',
description: 'A directory of the volume, relative to its root'
})
.example(
'$0 deploy volumes ls wp-content/mu-plugins -a my-blog',
'The mu-plugins of a WordPress app'
)
.showHelpOnFail(false) as any,
handler: async args => {
const sub = volume_dest(args.path)
const { api, team, app, volume } = await resolve_volume(
args.app,
args.volume
)
const op = await api.listVolumeFiles(
team,
volume.id,
sub === '.' ? undefined : sub
)
const done = require_completed(await wait_operation(api, team, op))
const entries = done.entries ?? []
if (args.json) return print_json({ path: sub, entries })
const where = `${app.name}/${volume.name}:${sub === '.' ? '/' : sub}`
if (entries.length === 0) {
log.info(`📂 ${where} is empty.`)
return
}
log.info(`📂 ${where}`)
for (const e of entries) print(` ${format_entry(e)}`)
if (entries.length >= 1000)
log.info('Only the first 1000 entries, by name.')
}
}
Loading
Loading