/* * Silex website builder, free/libre no-code tool for makers. * Copyright (c) 2023 lexoyo and Silex Labs foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ import { API_CONNECTOR_LOGIN_CALLBACK, API_CONNECTOR_PATH, API_PATH, WEBSITE_DATA_FILE, WEBSITE_PAGES_FOLDER } from '../../constants' import { ServerConfig } from '../../server/config' import { ConnectorFile, ConnectorFileContent, StatusCallback, StorageConnector, contentToBuffer, contentToString, toConnectorData } from '../../server/connectors/connectors' import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus, Page } from '../../types' import fetch from 'node-fetch' import crypto, { createHash } from 'crypto' import { join } from 'path' import { Agent } from 'https' import { getPageSlug } from '../../page' import e from 'express' import { fork } from 'child_process' /** * Gitlab connector * @fileoverview Gitlab connector for Silex, connect to the user's Gitlab account to store websites * @see https://docs.gitlab.com/ee/api/oauth2.html */ const MAX_BATCH_UPLOAD_SIZE = 100 const MAX_BODY_SIZE_KB = 8 * 1000 * 1024 // 8MB (note that 10 MB PNG → becomes ~13.3 MB → ❌ often too big for Gitlab) const WEBSITE_DATA_FILE_FORMAT_VERSION = '1.0.0' export interface GitlabOptions { clientId: string clientSecret: string branch: string assetsFolder: string repoPrefix: string scope: string domain: string timeOut: number //metaRepo: string //metaRepoFile: string } export interface GitlabToken { state?: string codeVerifier?: string codeChallenge?: string token?: { access_token: string token_type: string expires_in: number refresh_token: string created_at: number id_token: string scope: string } userId?: number username?: string } export type GitlabSession = Record interface GitlabAction { action: 'create' | 'delete' | 'move' | 'update' | 'cherry-pick' file_path?: string content?: string commit_id?: string encoding?: 'base64' | 'text' } interface GitlabWriteFile { branch: string commit_message: string id?: string actions?: GitlabAction[] content?: string file_path?: string encoding?: 'base64' | 'text' } interface GitlabGetToken { grant_type: 'authorization_code' client_id: string client_secret: string code: string redirect_uri: string code_verifier: string } interface GitlabWebsiteName { name: string } interface GitlabCreateBranch { branch: string ref: string } interface GitlabGetTags { per_page?: number } interface GitlabCreateTag { tag_name: string ref: string message: string } interface GitlabFetchCommits { ref_name: string since: string } interface GitlabPage { id: string name: string isFile: true } interface GitlabWebsiteData { fileFormatVersion: string pages: GitlabPage[] } // interface MetaRepoFileContent { // websites: { // [websiteId: string]: { // meta: WebsiteMetaFileContent, // createdAt: string, // updatedAt: string, // } // } // } const svg = ` ` const encodedSvg = encodeURIComponent(svg) const ICON = '/assets/gitlab.png' export function computeGitBlobSha(content: string, binary: boolean): string { const contentBuffer = binary ? Buffer.from(content, 'base64') // for binary files : Buffer.from(content, 'utf8') // for text files const header = `blob ${contentBuffer.length}\0` const full = Buffer.concat([Buffer.from(header), contentBuffer as Buffer]) return crypto.createHash('sha1').update(full).digest('hex') } function sanitizeGitlabPath(name: string): string { return name .normalize('NFD') // separate accents .replace(/[\u0300-\u036f]/g, '') // remove accents .replace(/[^a-zA-Z0-9._-]/g, '-') // only allow allowed characters .replace(/^[-_.]+/, '') // no starting '-', '_' or '.' .replace(/[-_.]+$/, '') // no ending '-', '_' or '.' .replace(/\.git$|\.atom$/i, '') // forbidden endings .toLowerCase() } export default class GitlabConnector implements StorageConnector { connectorId = 'gitlab' connectorType = ConnectorType.STORAGE displayName = 'GitLab' icon = ICON disableLogout = false color = '#2B1B63' background = 'rgba(252, 109, 38, 0.2)' options: GitlabOptions constructor(private config: ServerConfig, opts: Partial) { this.options = { branch: 'main', assetsFolder: 'assets', //metaRepo: 'silex-meta', //metaRepoFile: 'websites.json', repoPrefix: 'silex_', scope: 'api', // 'api+read_api+read_user+read_repository+write_repository+email+sudo+profile+openid' ...opts, } as GitlabOptions if (!this.options.clientId) throw new Error('Missing Gitlab client ID') if (!this.options.clientSecret) throw new Error('Missing Gitlab client secret') if (!this.options.domain) throw new Error('Missing Gitlab domain') if (!this.options.timeOut) this.options.timeOut = 15000 /* default value */ } // ** // Convenience methods for the Gitlab API private getAssetPath(path: string, encode = true): string { const resolvedPath = join(this.options.assetsFolder, path) if (encode) return encodeURIComponent(resolvedPath) return resolvedPath } isUsingOfficialInstance(): boolean { const gitlabDomainRegexp = /(^|\b)(gitlab\.com)($|\b)/ return gitlabDomainRegexp.test(this.options.domain) } async createFile(session: GitlabSession, websiteId: WebsiteId, path: string, content: string, isBase64 = false): Promise { // Remove leading slash const safePath = path.replace(/^\//, '') const encodePath = decodeURIComponent(path) return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/files/${safePath}`, method: 'POST', requestBody: { id: websiteId, branch: this.options.branch, content, commit_message: `Create file ${encodePath} from Silex`, encoding: isBase64 ? 'base64' : undefined, } }) } async updateFile(session: GitlabSession, websiteId: WebsiteId, path: string, content: string, isBase64 = false): Promise { // Remove leading slash const safePath = path.replace(/^\//, '') const encodePath = decodeURIComponent(path) return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/files/${safePath}`, method: 'PUT', requestBody: { id: websiteId, branch: this.options.branch, content: await contentToString(content), commit_message: `Update website asset ${encodePath} from Silex`, encoding: isBase64 ? 'base64' : undefined, } }) } async readFile(session: GitlabSession, websiteId: string, fileName: string): Promise { // Remove leading slash const safePath = fileName.replace(/^\//, '') return this.downloadRawFile(session, websiteId, safePath) } /** * Call the Gitlab API with the user's token and handle errors */ async callApi({ session, path, method = 'GET', requestBody = null, params = {}, responseHeaders = {}, // Will get the response heaaders }: { session: GitlabSession, path: string, method?: 'POST' | 'GET' | 'PUT' | 'DELETE', requestBody?: GitlabWriteFile | GitlabGetToken | GitlabWebsiteName | GitlabCreateBranch | GitlabGetTags | GitlabCreateTag | GitlabFetchCommits | null, params?: any, responseHeaders?: any, }): Promise { const token = this.getSessionToken(session).token const tokenParam = token ? `access_token=${token.access_token}&` : '' const paramsStr = Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent((v as any).toString())}`).join('&') const url = `${this.options.domain}/${path}?${tokenParam}${paramsStr}` const headers = { 'Content-Type': 'application/json', } if (method === 'GET' && requestBody) { console.error('Gitlab API error (4) - GET request with body', { url, method, body: requestBody, params }) } // With or without body let response const body = requestBody ? JSON.stringify(requestBody) : undefined try { if(body && Buffer.byteLength(body) > MAX_BODY_SIZE_KB * 1024) { // TODO: warn the end user console.warn('Gitlab API warning - body too big', Buffer.byteLength(body), 'bytes', { url, method, params }) } response = await fetch(url, requestBody && method !== 'GET' ? { agent: this.getAgent(), method, headers, body, } : { agent: this.getAgent(), method, headers, }) } catch (e) { console.error('Gitlab API error (0)', e) throw new ApiError(`Gitlab API error (0): ${e.message} ${e.text} ${e.code} ${e.name} ${e.type}`, 500) } // Pass the response headers to the caller response.headers.forEach((value, name) => responseHeaders[name] = value) // Handle the case when the server returns a non-JSON response (e.g. 400 Bad Request) const text = await async function () { try { return await response.text() } catch (e) { console.error('Gitlab API error (6) - could not parse response', response.status, response.statusText, { url, method, body: requestBody, params }, e) throw new ApiError(`Gitlab API error (6): response body not available. ${e.message}`, 500) } }() if (!response.ok) { console.error('Gitlab API error (7) - response not ok', response.status, response.statusText, { url, method, body: requestBody, params, text: text }) if (text.includes('A file with this name doesn\'t exist')) { throw new ApiError('Gitlab API error (5): Not Found', 404) } else if (response.status === 401 && this.getSessionToken(session).token?.refresh_token) { // Refresh the token const token = this.getSessionToken(session).token const body = { grant_type: 'refresh_token', refresh_token: token?.refresh_token, client_id: this.options.clientId, client_secret: this.options.clientSecret, } const response = await fetch(this.options.domain + '/oauth/token', { agent: this.getAgent(), method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(body) }) const refreshJson = await response.json() if (response.ok) { this.setSessionToken(session, { token: { ...token, ...refreshJson, }, } as GitlabToken) return await this.callApi({ session, path, method, requestBody: body as any, params, responseHeaders, }) } else { const message = typeof refreshJson?.message === 'object' ? Object.entries(refreshJson.message).map(entry => entry.join(' ')).join(' ') : refreshJson?.message ?? refreshJson?.error ?? response.statusText console.error('Gitlab API error (2) - could not refresh token', response.status, response.statusText, { message }, 'refresh_token:', token?.refresh_token) // Workaround for when the token is invalid // It happens often which is not normal (refresh token should last 6 months) this.logout(session) // Notify the user throw new ApiError(`Gitlab API error (2): ${message}`, response.status) } } else { const message = response.statusText console.error('Gitlab API error (1)', response.status, response.statusText, { url, method, body: requestBody, params, text: text, message }) throw new ApiError(`Gitlab API error (1): ${message} (${text})`, response.status) } } let json: { message: string, error: string } | any try { json = JSON.parse(text) } catch (e) { if (!response.ok) { // A real error throw e } else { // Useless error linked to the fact that the response is not JSON console.error('Gitlab API error (3) - could not parse response', response.status, response.statusText, { url, method, body: requestBody, params, text: text }) return text } } return json } async downloadRawFile(session: GitlabSession, projectId: string, filePath: string): Promise { const token = this.getSessionToken(session).token?.access_token const domain = this.options.domain const branch = this.options.branch // Construct the raw URL // GET /projects/:id/repository/files/:file_path/raw const rawUrl = `${domain}/api/v4/projects/${projectId}/repository/files/${encodeURIComponent(filePath)}/raw?ref=${branch}&access_token=${token}` const fileRes = await fetch(rawUrl, { agent: this.getAgent(), }) const contentType = fileRes.headers.get('content-type') if (contentType?.includes('text/html')) { const html = await fileRes.text() throw new ApiError('GitLab returned HTML instead of file (unauthorized or not found).', 401) } if (!fileRes.ok) { const errText = await fileRes.text() if (errText.includes('not found') || fileRes.status === 404) { throw new ApiError('GitLab raw error (5): Not Found', 404) } console.error('GitLab raw error (1)', fileRes.status, fileRes.statusText, { rawUrl, errText }) throw new ApiError(`GitLab raw error (1): ${fileRes.statusText} (${errText})`, fileRes.status) } try { const buffer = await fileRes.buffer() return buffer } catch (e) { console.error('GitLab raw error (3): could not read buffer', e) throw new ApiError('GitLab raw error (3): failed to read binary content', 500) } } private generateCodeVerifier() { return crypto.randomBytes(64).toString('base64') .replace(/\+/g, '-') .replace(/\//g, '_') .replace(/=/g, '') .substr(0, 128) } private async generateCodeChallenge(verifier) { const hashed = createHash('sha256').update(verifier).digest() let base64Url = hashed.toString('base64') // Replace '+' with '-', '/' with '_', and remove '=' base64Url = base64Url.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '') return base64Url } private getRedirect() { const params = `connectorId=${this.connectorId}&type=${this.connectorType}` return `${this.config.url}${API_PATH}${API_CONNECTOR_PATH}${API_CONNECTOR_LOGIN_CALLBACK}?${params}` } // Force IPv4 when running locally private getAgent(): Agent | undefined { if (this.config.url.startsWith('http://localhost')) { return new Agent({ family: 4, }) } return undefined } /** * Get the OAuth URL to redirect the user to * The URL should look like * https://gitlab.example.com/oauth/authorize?client_id=APP_ID&redirect_uri=REDIRECT_URI&response_type=code&state=STATE&scope=REQUESTED_SCOPES&code_challenge=CODE_CHALLENGE&code_challenge_method=S256 * OAuth2 Step #1 from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-with-proof-key-for-code-exchange-pkce */ async getOAuthUrl(session: GitlabSession): Promise { const redirect_uri = encodeURIComponent(this.getRedirect()) const state = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) const codeVerifier = this.generateCodeVerifier() // Create the code challenge const codeChallenge = await this.generateCodeChallenge(codeVerifier) // Store the code verifier and code challenge in the session this.setSessionToken(session, { ...this.getSessionToken(session), state, codeVerifier, codeChallenge, }) return `${this.options.domain}/oauth/authorize?client_id=${this.options.clientId}&redirect_uri=${redirect_uri}&response_type=code&state=${this.getSessionToken(session).state}&scope=${this.options.scope}&code_challenge=${codeChallenge}&code_challenge_method=S256` } getSessionToken(session: GitlabSession | undefined): GitlabToken { return (session ?? {})[this.connectorId] ?? {} } setSessionToken(session: GitlabSession, token: GitlabToken): void { session[this.connectorId] = token } resetSessionToken(session: GitlabSession): void { delete session[this.connectorId] } getOptions(formData: object): object { return {} // FIXME: store branch } async getLoginForm(session: GitlabSession, redirectTo: string): Promise { return null } async getSettingsForm(session: GitlabSession, redirectTo: string): Promise { return null } async isLoggedIn(session: GitlabSession): Promise { return !!this.getSessionToken(session).token } /** * Get the token from return code * Set the token in the session * OAuth2 Step #2 from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-with-proof-key-for-code-exchange-pkce */ async setToken(session: GitlabSession, loginResult: any): Promise { const sessionToken = this.getSessionToken(session) if (!loginResult.state || loginResult.state !== sessionToken?.state) { this.logout(session) throw new ApiError('Invalid state', 401) } if (!sessionToken?.codeVerifier) { this.logout(session) throw new ApiError('Missing code verifier', 401) } if (!sessionToken?.codeChallenge) { this.logout(session) throw new ApiError('Missing code challenge', 401) } const response = await fetch(this.options.domain + '/oauth/token', { agent: this.getAgent(), method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ client_id: this.options.clientId, client_secret: this.options.clientSecret, code: loginResult.code, grant_type: 'authorization_code', redirect_uri: this.getRedirect(), code_verifier: sessionToken.codeVerifier, }), }) const token = await response.json() // Store the token in the session this.setSessionToken(session, { ...this.getSessionToken(session), token, }) // We need to get the user ID for listWebsites const user = await this.callApi({ session, path: 'api/v4/user' }) as any // Store the user details in the session this.setSessionToken(session, { ...this.getSessionToken(session), userId: user.id, username: user.username, }) } async logout(session: GitlabSession): Promise { this.resetSessionToken(session) } async getUser(session: GitlabSession): Promise { const user = await this.callApi({ session, path: 'api/v4/user' }) as any return { name: user.name, email: user.email, picture: user.avatar_url, storage: await toConnectorData(session, this as StorageConnector), } } async listWebsites(session: GitlabSession): Promise { const userId = this.getSessionToken(session).userId if (!userId) { this.logout(session) throw new ApiError('Missing Gitlab user ID. User not logged in?', 401) } // Handle multiple pages let page = 1 let totalPages = 1 const projects: any[] = [] do { const responseHeaders: any = {} const pageProjects = await this.callApi({ session, path: `api/v4/users/${userId}/projects`, params: { simple: true, per_page: 100, page, }, responseHeaders, }) as any[] projects.push(...pageProjects) page++ // Get the total number of pages from the response headers const total = responseHeaders['x-total-pages'] if (total) { totalPages = parseInt(total, 10) } } while (page <= totalPages) return projects .filter(p => p.name.startsWith(this.options.repoPrefix)) .map(p => ({ websiteId: p.id, name: p.name.replace(this.options.repoPrefix, ''), createdAt: p.created_at, updatedAt: p.last_activity_at, connectorUserSettings: {}, })) } /** * Read the website data * The website data file is named `website.json` and the pages are named `page-{id}.json` * The pages are stored in the `src` folder by default */ async readWebsite(session: GitlabSession, websiteId: string): Promise { const websiteDataBuf = await this.downloadRawFile(session, websiteId, WEBSITE_DATA_FILE) const websiteDataOb = JSON.parse(websiteDataBuf.toString('utf8')) as GitlabWebsiteData & WebsiteData const { fileFormatVersion, ...websiteData } = websiteDataOb // Check the file format version if (fileFormatVersion !== WEBSITE_DATA_FILE_FORMAT_VERSION) { // This should be handled by a migration mechanism console.warn('Gitlab connector: website data file format version mismatch', fileFormatVersion, '!=', WEBSITE_DATA_FILE_FORMAT_VERSION) } // This happens when the website was just created // Let grapesjs create the pages in the frontend if (!websiteData.pages) { return websiteData as WebsiteData } // Load each page in parallel const pages = await Promise.all(websiteData.pages.map(async (page: GitlabPage | Page) => { if ((page as GitlabPage).isFile) { const name = getPageSlug(page.name) const fileName = (`${name}-${page.id}`) const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json` const pageContent = await this.downloadRawFile(session, websiteId, filePath) const res = JSON.parse(pageContent.toString('utf8')) as Page return res } return page as Page })) // Read each page file if needed return { ...websiteData, pages, } as WebsiteData } /** * Create a new website, i.e. a new Gitlab repository with an empty website data file */ async createWebsite(session: GitlabSession, websiteMeta: WebsiteMetaFileContent): Promise { const project = await this.callApi({ session, path: 'api/v4/projects/', method: 'POST', requestBody: { name: this.options.repoPrefix + websiteMeta.name, } }) as any await this.createFile(session, project.id, WEBSITE_DATA_FILE, JSON.stringify({} as WebsiteData)) //await this.createFile(session, project.id, WEBSITE_META_DATA_FILE, JSON.stringify(websiteMeta)) //await this.updateWebsite(session, project.id, {} as WebsiteData) //await this.setWebsiteMeta(session, project.id, websiteMeta) return project.id } /** * Update the website data * Split the website data into 1 file per page + 1 file for the website data itself * Use gitlab batch API to create/update the files */ async updateWebsite(session: GitlabSession, websiteId: WebsiteId, websiteData: WebsiteData): Promise { const batchActions: GitlabAction[] = [] // ** // List existing files in the pages folder const existingFiles = await this.ls({ session, websiteId, path: WEBSITE_PAGES_FOLDER, recursive: false, }) // ** // Create the pages for batch upload const pages = websiteData.pages.map((page: Page) => { const fileName = encodeURIComponent(`${getPageSlug(page.name)}-${page.id}`) const filePath = `${WEBSITE_PAGES_FOLDER}/${fileName}.json` const content = JSON.stringify(page) const newSha = computeGitBlobSha(content, false) const existingSha = existingFiles.get(filePath) if (existingSha) { if (existingSha !== newSha) { batchActions.push({ action: 'update', file_path: filePath, content, }) } // else: skip unchanged file } else { batchActions.push({ action: 'create', file_path: filePath, content, }) } return { name: page.name, id: page.id, isFile: true, } }) // ** // Delete pages that are not in the new website data for (const filePath of existingFiles.keys()) { const pageName = filePath.replace(/.*\//, '').replace(/\.json$/, '') const pageId = pageName.split('-').pop() const page = websiteData.pages.find((p: Page) => p.id === pageId) if (!page) { batchActions.push({ action: 'delete', file_path: filePath, }) } } // ** // Add the main website data file const websiteDataWithGitlabPages = { ...websiteData, fileFormatVersion: WEBSITE_DATA_FILE_FORMAT_VERSION, pages, } as GitlabWebsiteData const websiteJsonContent = JSON.stringify(websiteDataWithGitlabPages) batchActions.push({ action: 'update', file_path: WEBSITE_DATA_FILE, content: websiteJsonContent, }) // ** // Perform a batch commit return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { branch: this.options.branch, commit_message: 'Update website data from Silex', actions: batchActions, }, }) } async deleteWebsite(session: GitlabSession, websiteId: WebsiteId): Promise { // Delete repo await this.callApi({ session, path: `api/v4/projects/${websiteId}`, method: 'DELETE' }) //// Load the meta repo data //const file = await this.callApi(session, `api/v4/projects/${this.getMetaRepoPath(session)}/repository/files/${this.options.metaRepoFile}`, 'GET', null, { // ref: this.options.branch, //}) //const metaRepo = JSON.parse(Buffer.from(file.content, 'base64').toString('utf8')) as MetaRepoFileContent //const data = metaRepo.websites[websiteId] //if(!data) throw new ApiError(`Website ${websiteId} not found`, 404) //// Update or create the website meta data //delete metaRepo.websites[websiteId] //// Save the meta repo data //const project = await this.callApi(session, `api/v4/projects/${this.getMetaRepoPath(session)}/repository/files/${this.options.metaRepoFile}`, 'PUT', { // branch: this.options.branch, // commit_message: `Delete meta data of ${data.meta.name} (${websiteId}) from Silex`, // content: JSON.stringify(metaRepo), //}) } // async duplicateWebsite(session: GitlabSession, websiteId: string): Promise { // // Get the repo meta data // const meta = await this.getWebsiteMeta(session, websiteId) // // List all the repository files // const files = await this.ls({ // session, // websiteId, // recursive: true, // }) // // Create a new repo // const newId = await this.createWebsite(session, { // ...meta, // name: meta.name + ' Copy ' + new Date().toISOString().replace(/T.*/, '') + ' ' + Math.random().toString(36).substring(2, 4), // }) // // Upload all files // const actions: GitlabAction[] = [] // for (const file of files.keys()) { // const content = await this.readFile(session, websiteId, file) // // From buffer to string // const contentStr = content.toString('base64') // const path = encodeURIComponent(file) // switch (file) { // case WEBSITE_DATA_FILE: // actions.push({ // action: 'update', // file_path: path, // content: contentStr, // }) // break // default: // actions.push({ // action: 'create', // file_path: path, // content: contentStr, // }) // } // } // // Perform a batch commit // return this.callApi({ // session, // path: `api/v4/projects/${newId}/repository/commits`, // method: 'POST', // requestBody: { // branch: this.options.branch, // commit_message: 'Update website data from Silex', // actions, // }, // }) // } // Fork the repo async duplicateWebsite(session: GitlabSession, websiteId: string): Promise { const meta = await this.getWebsiteMeta(session, websiteId) const forkName = `${meta.name} Copy ${new Date().toISOString().slice(0, 10)} ${Math.random().toString(36).substring(2, 4)}` const safePath = sanitizeGitlabPath(forkName) const forkedProject = await this.callApi({ session, path: `api/v4/projects/${websiteId}/fork`, method: 'POST', requestBody: { name: this.options.repoPrefix + forkName, /* @ts-ignore */ path: safePath, /* @ts-ignore */ namespace: meta.namespace?.id || undefined, }, }) return forkedProject.id } async getWebsiteMeta(session: GitlabSession, websiteId: WebsiteId): Promise { const project = await this.callApi({ session, path: `api/v4/projects/${websiteId}` }) return { websiteId, name: project.name.replace(this.options.repoPrefix, ''), imageUrl: project.avatar_url, createdAt: project.created_at, updatedAt: project.last_activity_at, connectorUserSettings: {}, } } async setWebsiteMeta(session: GitlabSession, websiteId: WebsiteId, websiteMeta: WebsiteMetaFileContent): Promise { // Rename the repo if needed const oldMeta = await this.getWebsiteMeta(session, websiteId) if (websiteMeta.name !== oldMeta.name) { await this.callApi({ session, path: `api/v4/projects/${websiteId}`, method: 'PUT', requestBody: { name: this.options.repoPrefix + websiteMeta.name, } }) } } async writeAssets(session: GitlabSession, websiteId: string, files: ConnectorFile[], status?: StatusCallback, removeUnlisted = false): Promise { status && await status({ message: `Preparing ${files.length} files`, status: JobStatus.IN_PROGRESS }) // List all the files in assets folder const existingFiles = await this.ls({ session, websiteId, recursive: true, path: this.options.assetsFolder, }) // Create the actions for the batch const filesToUpload = [] as GitlabAction[] const filesToKeep = new Set(files.map(file => this.getAssetPath(file.path, false))) for (const file of files) { const filePath = this.getAssetPath(file.path, false) const content = (await contentToBuffer(file.content)).toString('base64') const existingSha = existingFiles.get(filePath) const newSha = computeGitBlobSha(content, true) if (existingSha) { if (existingSha !== newSha) { filesToUpload.push({ action: 'update', file_path: filePath, content, encoding: 'base64', }) } // else: skip unchanged file } else { filesToUpload.push({ action: 'create', file_path: filePath, content, encoding: 'base64', }) } } // Optionally remove unlisted files if (removeUnlisted) { for (const [existingFilePath] of existingFiles) { if (!filesToKeep.has(existingFilePath)) { filesToUpload.push({ action: 'delete', file_path: existingFilePath, }) } } } // Split the files into chunks to avoid the number of files limit const chunks: GitlabAction[][] = [] for (let i = 0; i < filesToUpload.length; i += MAX_BATCH_UPLOAD_SIZE) { chunks.push(filesToUpload.slice(i, i + MAX_BATCH_UPLOAD_SIZE)) } // Notify the user if nothing changed if (chunks.length === 0) { console.info('No files to upload') status && await status({ message: 'No files to upload', status: JobStatus.SUCCESS }) return } // Upload the files in chunks for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) { const chunk = chunks[chunkIndex] if (chunks.length > 1) { status && await status({ message: `Batch ${chunkIndex + 1}/${chunks.length}: Uploading ${chunk.length} files`, status: JobStatus.IN_PROGRESS }) } else { status && await status({ message: `Uploading ${files.length} file(s)`, status: JobStatus.IN_PROGRESS }) } try { await this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { branch: 'main', commit_message: `Batch update assets (${chunkIndex + 1}/${chunks.length})`, actions: chunk, } }) } catch (e) { console.error(`Batch ${chunkIndex + 1} failed`, e) status && await status({ message: `Error in batch ${chunkIndex + 1}`, status: JobStatus.ERROR }) throw e } } status && await status({ message: `All ${files.length} files uploaded`, status: JobStatus.SUCCESS }) } async readAsset(session: GitlabSession, websiteId: string, fileName: string): Promise { const finalPath = this.getAssetPath(fileName, false) return this.readFile(session, websiteId, finalPath) } async deleteAssets(session: GitlabSession, websiteId: string, fileNames: string[]): Promise { return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { id: websiteId, branch: this.options.branch, commit_message: `Delete assets from Silex: ${fileNames.join(', ')}`, actions: fileNames.map(f => ({ action: 'delete', file_path: this.getAssetPath(f), })), } }) } /* * Get the meta repo path for the current user * The meta repo contains a JSON file which contains the list of websites */ //private getMetaRepoPath(session: GitlabSession): string { // if(!this.getSessionToken(session).username) throw new ApiError('Missing Gitlab user ID. User not logged in?', 401) // return encodeURIComponent(`${this.getSessionToken(session).username}/${this.options.metaRepo}`) //} ///** // * Initialize the storage with a meta repo // */ //private async initStorage(session: GitlabSession): Promise { // // Create the meta repo // try { // const project = await this.callApi(session, 'api/v4/projects/', 'POST', { // name: this.options.metaRepo, // }) as any // return this.createFile(session, this.getMetaRepoPath(session), this.options.metaRepoFile, JSON.stringify({ // websites: {} // } as MetaRepoFileContent)) // } catch (e) { // console.error('Could not init storage', e.statusCode, e.httpStatusCode, e) // throw e // } //} /** * List all the files in a folder * The result is a map of file paths to their SHA */ protected async ls({ session, websiteId, recursive = false, path, }: { session: GitlabSession, websiteId: string, recursive?: boolean, path?: string, }): Promise> { const existingPaths = new Map() let page = 1 let keepGoing = true while (keepGoing) { const responseHeaders: any = {} let tree: any[] = [] try { const params = { recursive, per_page: 100, page, } as any if (path) params.path = path tree = await this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/tree`, method: 'GET', params, responseHeaders, }) } catch (e) { // Allow 404 errors // This happens when the folder does not exist // In git this just means the files don't exist yet if (e.statusCode !== 404 && e.httpStatusCode !== 404) { throw e } } // Filter the files tree .filter(item => item.type === 'blob') .forEach(item => existingPaths.set(item.path, item.id)) // Check if we need to keep going const maxPages = responseHeaders['x-total-pages'] ? parseInt(responseHeaders['x-total-pages'], 10) : 1 keepGoing = page < maxPages page++ } // Return the set of existing paths return existingPaths } }