/* * Silex website builder, free/libre no-code tool for makers. * Copyright (c) 2023 lexoyo and Silex Labs foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ import { API_CONNECTOR_LOGIN_CALLBACK, API_CONNECTOR_PATH, API_PATH, WEBSITE_DATA_FILE, LEGACY_WEBSITE_PAGES_FOLDER, WEBSITE_PAGES_FOLDER } from '../../constants' import { ServerConfig } from '../../server/config' import { ConnectorFile, ConnectorFileContent, StatusCallback, StorageConnector, contentToBuffer, contentToString, toConnectorData } from '../../server/connectors/connectors' import { ApiError, ConnectorType, ConnectorUser, WebsiteData, WebsiteId, WebsiteMeta, WebsiteMetaFileContent, JobStatus, EMPTY_WEBSITE } from '../../types' import fetch from 'node-fetch' import crypto, { createHash } from 'crypto' import { join } from 'path' import { Agent } from 'https' import { getPageSlug } from '../../page' import e from 'express' import { fork } from 'child_process' import { Page } from 'grapesjs' import { stringify, split, merge, getPagesFolder } from '../../server/utils/websiteDataSerialization' /** * Gitlab connector * @fileoverview Gitlab connector for Silex, connect to the user's Gitlab account to store websites * @see https://docs.gitlab.com/ee/api/oauth2.html */ const MAX_BATCH_UPLOAD_SIZE = 100 const MAX_BODY_SIZE_KB = 8 * 1000 * 1024 // 8MB (note that 10 MB PNG → becomes ~13.3 MB → ❌ often too big for Gitlab) const WEBSITE_DATA_FILE_FORMAT_VERSION = '1.0.0' export interface GitlabOptions { clientId: string clientSecret: string branch: string assetsFolder: string repoPrefix: string scope: string domain: string timeOut: number //metaRepo: string //metaRepoFile: string } export interface GitlabToken { state?: string codeVerifier?: string codeChallenge?: string token?: { access_token: string token_type: string expires_in: number refresh_token: string created_at: number id_token: string scope: string } userId?: number username?: string } export type GitlabSession = Record interface GitlabAction { action: 'create' | 'delete' | 'move' | 'update' | 'cherry-pick' file_path?: string content?: string commit_id?: string encoding?: 'base64' | 'text' } interface GitlabWriteFile { branch: string commit_message: string id?: string actions?: GitlabAction[] content?: string file_path?: string encoding?: 'base64' | 'text' } interface GitlabGetToken { grant_type: 'authorization_code' client_id: string client_secret: string code: string redirect_uri: string code_verifier: string } interface GitlabWebsiteName { name: string } interface GitlabCreateBranch { branch: string ref: string } interface GitlabGetTags { per_page?: number } interface GitlabCreateTag { tag_name: string ref: string message: string } interface GitlabFetchCommits { ref_name: string since: string } // interface MetaRepoFileContent { // websites: { // [websiteId: string]: { // meta: WebsiteMetaFileContent, // createdAt: string, // updatedAt: string, // } // } // } const svg = ` ` const encodedSvg = encodeURIComponent(svg) const ICON = '/assets/gitlab.png' export function computeGitBlobSha(content: string, binary: boolean): string { const contentBuffer = binary ? Buffer.from(content, 'base64') // for binary files : Buffer.from(content, 'utf8') // for text files const header = `blob ${contentBuffer.length}\0` const full = Buffer.concat([Buffer.from(header), contentBuffer as Buffer]) return crypto.createHash('sha1').update(full).digest('hex') } function sanitizeGitlabPath(name: string): string { return name .normalize('NFD') // separate accents .replace(/[\u0300-\u036f]/g, '') // remove accents .replace(/[^a-zA-Z0-9._-]/g, '-') // only allow allowed characters .replace(/^[-_.]+/, '') // no starting '-', '_' or '.' .replace(/[-_.]+$/, '') // no ending '-', '_' or '.' .replace(/\.git$|\.atom$/i, '') // forbidden endings .toLowerCase() } export default class GitlabConnector implements StorageConnector { connectorId = 'gitlab' connectorType = ConnectorType.STORAGE displayName = 'GitLab' icon = ICON disableLogout = false color = '#2B1B63' background = 'rgba(252, 109, 38, 0.2)' options: GitlabOptions constructor(private config: ServerConfig, opts: Partial) { this.options = { branch: 'main', assetsFolder: 'assets', //metaRepo: 'silex-meta', //metaRepoFile: 'websites.json', repoPrefix: 'silex_', scope: 'api', // 'api+read_api+read_user+read_repository+write_repository+email+sudo+profile+openid' ...opts, } as GitlabOptions if (!this.options.clientId) throw new Error('Missing Gitlab client ID') if (!this.options.clientSecret) throw new Error('Missing Gitlab client secret') if (!this.options.domain) throw new Error('Missing Gitlab domain') if (!this.options.timeOut) this.options.timeOut = 15000 /* default value */ } // ** // Convenience methods for the Gitlab API private getAssetPath(path: string, encode = true): string { const resolvedPath = join(this.options.assetsFolder, path) if (encode) return encodeURIComponent(resolvedPath) return resolvedPath } isUsingOfficialInstance(): boolean { const gitlabDomainRegexp = /(^|\b)(gitlab\.com)($|\b)/ return gitlabDomainRegexp.test(this.options.domain) } async createFile(session: GitlabSession, websiteId: WebsiteId, path: string, content: string, isBase64 = false): Promise { // Remove leading slash const safePath = path.replace(/^\//, '') const encodePath = decodeURIComponent(path) return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/files/${safePath}`, method: 'POST', requestBody: { id: websiteId, branch: this.options.branch, content, commit_message: `Create file ${encodePath} from Silex`, encoding: isBase64 ? 'base64' : undefined, } }) } async updateFile(session: GitlabSession, websiteId: WebsiteId, path: string, content: string, isBase64 = false): Promise { // Remove leading slash const safePath = path.replace(/^\//, '') const encodePath = decodeURIComponent(path) return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/files/${safePath}`, method: 'PUT', requestBody: { id: websiteId, branch: this.options.branch, content: await contentToString(content), commit_message: `Update website asset ${encodePath} from Silex`, encoding: isBase64 ? 'base64' : undefined, } }) } async readFile(session: GitlabSession, websiteId: string, fileName: string): Promise { // Remove leading slash const safePath = fileName.replace(/^\//, '') return this.downloadRawFile(session, websiteId, safePath) } /** * Call the Gitlab API with the user's token and handle errors */ async callApi({ session, path, method = 'GET', requestBody = null, params = {}, responseHeaders = {}, // Will get the response heaaders }: { session: GitlabSession, path: string, method?: 'POST' | 'GET' | 'PUT' | 'DELETE', requestBody?: GitlabWriteFile | GitlabGetToken | GitlabWebsiteName | GitlabCreateBranch | GitlabGetTags | GitlabCreateTag | GitlabFetchCommits | null, params?: any, responseHeaders?: any, }): Promise { const token = this.getSessionToken(session).token const tokenParam = token ? `access_token=${token.access_token}&` : '' const paramsStr = Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent((v as any).toString())}`).join('&') const url = `${this.options.domain}/${path}?${tokenParam}${paramsStr}` const headers = { 'Content-Type': 'application/json', } if (method === 'GET' && requestBody) { console.error('[GitlabConnector] Invalid GET request: GET requests should not have a body', { url, method, body: requestBody, params }) } // With or without body let response: Response const body = requestBody ? JSON.stringify(requestBody) : undefined try { if (body && Buffer.byteLength(body) > MAX_BODY_SIZE_KB * 1024) { console.warn('[GitlabConnector] Request body size exceeds Gitlab API limit', { size: Buffer.byteLength(body), maxAllowed: MAX_BODY_SIZE_KB * 1024, url, method, params, session, }) } response = await fetch(url, requestBody && method !== 'GET' ? { agent: this.getAgent(), method, headers, body, } : { agent: this.getAgent(), method, headers, }) } catch (e) { console.error('[GitlabConnector] Failed to reach Gitlab API endpoint', { error: e, url, method, body: requestBody, params, session, stack: e?.stack || new Error().stack, }) throw new ApiError(`Could not reach Gitlab API: ${e.message || e}`, 500) } // Pass the response headers to the caller response.headers.forEach((value, name) => responseHeaders[name] = value) // Handle the case when the server returns a non-JSON response (e.g. 400 Bad Request) const text = await async function () { try { return await response.text() } catch (e) { console.error('[GitlabConnector] Failed to read response body from Gitlab API', { status: response.status, statusText: response.statusText, url, method, body: requestBody, params, error: e }) throw new ApiError(`Gitlab API: Could not read response body (${e.message})`, 500) } }() if (!response.ok) { console.error('[GitlabConnector] Gitlab API responded with error status', { status: response.status, statusText: response.statusText, url, method, body: requestBody, params, responseText: text }) if (text.includes('A file with this name doesn\'t exist')) { throw new ApiError('Gitlab API: File not found', 404) } else if (response.status === 401 && this.getSessionToken(session).token?.refresh_token) { // Refresh the token const token = this.getSessionToken(session).token const body = { grant_type: 'refresh_token', refresh_token: token?.refresh_token, client_id: this.options.clientId, client_secret: this.options.clientSecret, } const response = await fetch(this.options.domain + '/oauth/token', { agent: this.getAgent(), method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(body) }) const refreshJson = await response.json() if (response.ok) { this.setSessionToken(session, { token: { ...token, ...refreshJson, }, } as GitlabToken) return await this.callApi({ session, path, method, requestBody: body as any, params, responseHeaders, }) } else { const message = typeof refreshJson?.message === 'object' ? Object.entries(refreshJson.message).map(entry => entry.join(' ')).join(' ') : refreshJson?.message ?? refreshJson?.error ?? response.statusText console.error('[GitlabConnector] Failed to refresh Gitlab OAuth token', { status: response.status, statusText: response.statusText, message, refresh_token: token?.refresh_token }) this.logout(session) throw new ApiError(`Gitlab API: Could not refresh token (${message})`, response.status) } } else { const message = response.statusText console.error('[GitlabConnector] Unhandled Gitlab API error response', { status: response.status, statusText: response.statusText, url, method, body: requestBody, params, responseText: text, message }) throw new ApiError(`Gitlab API error: ${message} (${text})`, response.status) } } let json: { message: string, error: string } | any try { json = JSON.parse(text) } catch (e) { if (!response.ok) { throw e } else { console.error('[GitlabConnector] Response from Gitlab API is not valid JSON', { statusText: response.statusText, url, method, body: requestBody, params, responseText: text, error: e, session, stack: e?.stack || new Error().stack, }) return text } } return json } async downloadRawFile(session: GitlabSession, projectId: string, filePath: string): Promise { const token = this.getSessionToken(session).token?.access_token const domain = this.options.domain const branch = this.options.branch // Construct the raw URL // GET /projects/:id/repository/files/:file_path/raw const rawUrl = `${domain}/api/v4/projects/${projectId}/repository/files/${encodeURIComponent(filePath)}/raw?ref=${branch}&access_token=${token}` const fileRes = await fetch(rawUrl, { agent: this.getAgent(), }) const contentType = fileRes.headers.get('content-type') if (contentType?.includes('text/html')) { const html = await fileRes.text() throw new ApiError('GitLab returned HTML instead of file (unauthorized or not found).', 401) } if (!fileRes.ok) { const errText = await fileRes.text() if (errText.includes('not found') || fileRes.status === 404) { console.error('[GitlabConnector] GitLab raw file not found', { status: fileRes.status, statusText: fileRes.statusText, url: rawUrl, responseText: errText }) throw new ApiError(`GitLab raw file not found: filePath="${filePath}" (status ${fileRes.status})`, 404) } console.error('[GitlabConnector] Failed to fetch raw file from GitLab', { filePath, url: rawUrl, status: fileRes.status, statusText: fileRes.statusText, responseText: errText }) throw new ApiError(`Failed to fetch raw file from GitLab: filePath="${filePath}" - ${fileRes.statusText} (${errText})`, fileRes.status) } try { const buffer = await fileRes.buffer() return buffer } catch (e) { console.error('[GitlabConnector] Error reading binary content from GitLab raw file', e) throw new ApiError('Failed to read binary content from GitLab raw file', 500) } } private generateCodeVerifier() { return crypto.randomBytes(64).toString('base64') .replace(/\+/g, '-') .replace(/\//g, '_') .replace(/=/g, '') .substr(0, 128) } private async generateCodeChallenge(verifier) { const hashed = createHash('sha256').update(verifier).digest() let base64Url = hashed.toString('base64') // Replace '+' with '-', '/' with '_', and remove '=' base64Url = base64Url.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '') return base64Url } private getRedirect() { const params = `connectorId=${this.connectorId}&type=${this.connectorType}` return `${this.config.url}${API_PATH}${API_CONNECTOR_PATH}${API_CONNECTOR_LOGIN_CALLBACK}?${params}` } // Force IPv4 when running locally private getAgent(): Agent | undefined { if (this.config.url.startsWith('http://localhost')) { return new Agent({ family: 4, }) } return undefined } /** * Get the OAuth URL to redirect the user to * The URL should look like * https://gitlab.example.com/oauth/authorize?client_id=APP_ID&redirect_uri=REDIRECT_URI&response_type=code&state=STATE&scope=REQUESTED_SCOPES&code_challenge=CODE_CHALLENGE&code_challenge_method=S256 * OAuth2 Step #1 from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-with-proof-key-for-code-exchange-pkce */ async getOAuthUrl(session: GitlabSession): Promise { const redirect_uri = encodeURIComponent(this.getRedirect()) const state = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) const codeVerifier = this.generateCodeVerifier() // Create the code challenge const codeChallenge = await this.generateCodeChallenge(codeVerifier) // Store the code verifier and code challenge in the session this.setSessionToken(session, { ...this.getSessionToken(session), state, codeVerifier, codeChallenge, }) return `${this.options.domain}/oauth/authorize?client_id=${this.options.clientId}&redirect_uri=${redirect_uri}&response_type=code&state=${this.getSessionToken(session).state}&scope=${this.options.scope}&code_challenge=${codeChallenge}&code_challenge_method=S256` } getSessionToken(session: GitlabSession | undefined): GitlabToken { return (session ?? {})[this.connectorId] ?? {} } setSessionToken(session: GitlabSession, token: GitlabToken): void { session[this.connectorId] = token } resetSessionToken(session: GitlabSession): void { delete session[this.connectorId] } getOptions(formData: object): object { return {} // FIXME: store branch } async getLoginForm(session: GitlabSession, redirectTo: string): Promise { return null } async getSettingsForm(session: GitlabSession, redirectTo: string): Promise { return null } async isLoggedIn(session: GitlabSession): Promise { return !!this.getSessionToken(session).token } /** * Get the token from return code * Set the token in the session * OAuth2 Step #2 from https://docs.gitlab.com/ee/api/oauth2.html#authorization-code-with-proof-key-for-code-exchange-pkce */ async setToken(session: GitlabSession, loginResult: any): Promise { const sessionToken = this.getSessionToken(session) // Handle state that may be JSON with redirect info or a plain string // Redirect info is when the user is comming from the /fork/ page of the dashboard let receivedState = loginResult.state try { const parsed = JSON.parse(loginResult.state) if (parsed.state) { receivedState = parsed.state } } catch { // Plain text } if (!receivedState || receivedState !== sessionToken?.state) { this.logout(session) throw new ApiError('Invalid state', 401) } if (!sessionToken?.codeVerifier) { this.logout(session) throw new ApiError('Missing code verifier', 401) } if (!sessionToken?.codeChallenge) { this.logout(session) throw new ApiError('Missing code challenge', 401) } const response = await fetch(this.options.domain + '/oauth/token', { agent: this.getAgent(), method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ client_id: this.options.clientId, client_secret: this.options.clientSecret, code: loginResult.code, grant_type: 'authorization_code', redirect_uri: this.getRedirect(), code_verifier: sessionToken.codeVerifier, }), }) const token = await response.json() // Store the token in the session this.setSessionToken(session, { ...this.getSessionToken(session), token, }) // We need to get the user ID for listWebsites const user = await this.callApi({ session, path: 'api/v4/user' }) as any // Store the user details in the session this.setSessionToken(session, { ...this.getSessionToken(session), userId: user.id, username: user.username, }) } async logout(session: GitlabSession): Promise { this.resetSessionToken(session) } async getUser(session: GitlabSession): Promise { const user = await this.callApi({ session, path: 'api/v4/user' }) as any return { name: user.name, email: user.email, picture: user.avatar_url, storage: await toConnectorData(session, this as StorageConnector), } } async listWebsites(session: GitlabSession): Promise { const userId = this.getSessionToken(session).userId if (!userId) { this.logout(session) throw new ApiError('Missing Gitlab user ID. User not logged in?', 401) } // Handle multiple pages let page = 1 let totalPages = 1 const projects: any[] = [] do { const responseHeaders: any = {} const pageProjects = await this.callApi({ session, path: `api/v4/users/${userId}/projects`, params: { per_page: 100, page, }, responseHeaders, }) as any[] projects.push(...pageProjects) page++ // Get the total number of pages from the response headers const total = responseHeaders['x-total-pages'] if (total) { totalPages = parseInt(total, 10) } } while (page <= totalPages) return projects .filter( p => p.name.startsWith(this.options.repoPrefix) && !p.marked_for_deletion_on && !p.marked_for_deletion_at ) .map(p => ({ websiteId: p.id, name: p.name.replace(this.options.repoPrefix, ''), createdAt: p.created_at, updatedAt: p.last_activity_at, connectorUserSettings: {}, })) } /** * Read the website data * The website data file is named `website.json` and the pages are named `page-{id}.json` * The pages are stored in the `src` folder by default */ async readWebsite(session: GitlabSession, websiteId: string): Promise { const websiteDataBuf = await this.downloadRawFile(session, websiteId, WEBSITE_DATA_FILE) const websiteDataContent = websiteDataBuf.toString('utf8') // Use the common merge function to reconstruct website data const pageLoader = async (pagePath: string): Promise => { const pageBuffer = await this.downloadRawFile(session, websiteId, pagePath) return pageBuffer.toString('utf8') } return await merge(websiteDataContent, pageLoader) } /** * Create a new website, i.e. a new Gitlab repository with an empty website data file */ async createWebsite(session: GitlabSession, websiteMeta: WebsiteMetaFileContent): Promise { const project = await this.callApi({ session, path: 'api/v4/projects/', method: 'POST', requestBody: { name: this.options.repoPrefix + websiteMeta.name, } }) as any await this.createFile(session, project.id, WEBSITE_DATA_FILE, stringify(EMPTY_WEBSITE)) //await this.createFile(session, project.id, WEBSITE_META_DATA_FILE, JSON.stringify(websiteMeta)) //await this.updateWebsite(session, project.id, {} as WebsiteData) //await this.setWebsiteMeta(session, project.id, websiteMeta) return project.id } /** * Update the website data * Split the website data into 1 file per page + 1 file for the website data itself * Use gitlab batch API to create/update the files */ async updateWebsite(session: GitlabSession, websiteId: WebsiteId, websiteData: WebsiteData): Promise { const batchActions: GitlabAction[] = [] // ** // Handle the legacy sites that have no pagesFolder // We want them to use "pages/" by default despite their files being in "src/" for now let isLegacySite = !websiteData.pagesFolder // ** // Backward compatibility case // The second time a legacy site is saved, the new `pages/` folder has been created // but the front end still doesn't have the `pagesFolder` key // This will be the case until the front end reloads if (isLegacySite) { const rootFiles = await this.ls({ session, websiteId, path: WEBSITE_PAGES_FOLDER, recursive: false, }) if (rootFiles.size) { // Here the new `pages/` folder has been created already isLegacySite = false websiteData.pagesFolder = WEBSITE_PAGES_FOLDER } } // ** // List existing files in the OLD pages folder (to detect files to delete) const existingFiles = await this.ls({ session, websiteId, path: getPagesFolder(websiteData), recursive: false, }) // ** // Force pagesFolder to 'pages' for writing if not defined if (isLegacySite) { websiteData.pagesFolder = WEBSITE_PAGES_FOLDER } // ** // Use the common split function to create files const filesToWrite = split(websiteData) // ** // Process each file to create/update for (const file of filesToWrite) { const filePath = file.path const content = file.content const newSha = computeGitBlobSha(content, false) const existingSha = existingFiles.get(filePath) || (filePath === WEBSITE_DATA_FILE ? 'always-update' : undefined) if (existingSha) { if (existingSha !== newSha) { batchActions.push({ action: 'update', file_path: filePath, content, }) } // else: skip unchanged file } else { batchActions.push({ action: 'create', file_path: filePath, content, }) } } // ** // Delete pages that are not in the new website data const pathsToWrite = filesToWrite.map(f => f.path) for (const filePath of existingFiles.keys()) { if (!pathsToWrite.includes(filePath)) { batchActions.push({ action: 'delete', file_path: filePath, }) } } // ** // Perform a batch commit return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { branch: this.options.branch, commit_message: 'Update website data from Silex', actions: batchActions, }, }) } async deleteWebsite(session: GitlabSession, websiteId: WebsiteId): Promise { // Delete repo await this.callApi({ session, path: `api/v4/projects/${websiteId}`, method: 'DELETE', }) } // Fork the repo (user's own project) async duplicateWebsite(session: GitlabSession, websiteId: string): Promise { const meta = await this.getWebsiteMeta(session, websiteId) const forkName = `${meta.name} Copy ${new Date().toISOString().slice(0, 10)} ${Math.random().toString(36).substring(2, 4)}` const safePath = sanitizeGitlabPath(forkName) const forkedProject = await this.callApi({ session, path: `api/v4/projects/${websiteId}/fork`, method: 'POST', requestBody: { name: this.options.repoPrefix + forkName, /* @ts-ignore */ path: safePath, /* @ts-ignore */ namespace: meta.namespace?.id || undefined, }, }) return forkedProject.id } /** * Fork an external/public GitLab project (from any user/organization) * @param session - The user session * @param gitlabUrl - The project path in the "username/repo" format * @returns The new website ID (project ID) */ async forkWebsite(session: GitlabSession, gitlabUrl: string): Promise { // Only accept "username/repo" pattern (no URLs) const projectPath = gitlabUrl.trim() if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(projectPath)) { throw new ApiError('Invalid project path. Use the "username/repo" format.', 400) } // URL-encode the project path for the API const encodedPath = encodeURIComponent(projectPath) // First, get the source project info to extract its name let sourceProject: any try { sourceProject = await this.callApi({ session, path: `api/v4/projects/${encodedPath}`, method: 'GET', }) } catch (e) { if (e.httpStatusCode === 404) { throw new ApiError(`Project not found: ${projectPath}. Make sure the project exists and is public or you have access to it.`, 404) } throw e } // Generate a unique name for the fork const sourceName = sourceProject.name.replace(this.options.repoPrefix, '') const forkName = `${sourceName} ${new Date().toISOString().slice(0, 10)} ${Math.random().toString(36).substring(2, 4)}` const safePath = sanitizeGitlabPath(this.options.repoPrefix + forkName) // Fork the project to the user's namespace const forkedProject = await this.callApi({ session, path: `api/v4/projects/${encodedPath}/fork`, method: 'POST', requestBody: { name: this.options.repoPrefix + forkName, /* @ts-ignore */ path: safePath, visibility: 'private', }, }) // Wait for the fork to complete (GitLab forks asynchronously) const forkedProjectId = forkedProject.id.toString() const maxAttempts = 30 // 30 attempts * 2 seconds = 60 seconds max const pollInterval = 2000 // 2 seconds for (let attempt = 0; attempt < maxAttempts; attempt++) { const project = await this.callApi({ session, path: `api/v4/projects/${forkedProjectId}`, method: 'GET', }) if (project.import_status === 'finished' || project.import_status === 'none') { return forkedProjectId } if (project.import_status === 'failed') { throw new ApiError(`Fork failed: ${project.import_error || 'Unknown error'}`, 500) } // Status is 'scheduled' or 'started', wait and retry await new Promise(resolve => setTimeout(resolve, pollInterval)) } return forkedProjectId } async getWebsiteMeta(session: GitlabSession, websiteId: WebsiteId): Promise { const project = await this.callApi({ session, path: `api/v4/projects/${websiteId}` }) // Defaults let pagesUrl: string | undefined = undefined let pagesVisibility: string | undefined = undefined let lastJob: { date: string, status: string, id?: number, name?: string, webUrl?: string } | undefined = undefined // Try to get GitLab Pages info try { const pages = await this.callApi({ session, path: `api/v4/projects/${websiteId}/pages` }) pagesUrl = pages.url // pages.access_control_level may be present; fallback to project.visibility if not pagesVisibility = pages.access_control_level || project.visibility } catch (e) { // If pages are not enabled or call fails, ignore and leave undefined } // Try to get last relevant job (prefer 'pages' job, otherwise most recent job) try { const jobs = await this.callApi({ session, path: `api/v4/projects/${websiteId}/jobs`, params: { per_page: 1, order_by: 'finished_at', sort: 'desc' } }) if (Array.isArray(jobs) && jobs.length > 0) { lastJob = { date: jobs[0].finished_at || jobs[0].created_at || jobs[0].started_at, status: jobs[0].status, id: jobs[0].id, name: jobs[0].name, webUrl: jobs[0].web_url } } } catch (e) { // With no jobs, leave lastJob undefined } // Fork info - fetch the forked-from project to get license let forkedFrom: { id: string, name: string, webUrl?: string, license?: string } | undefined = undefined if (project.forked_from_project) { let license: string | undefined = undefined try { const templateProject = await this.callApi({ session, path: `api/v4/projects/${project.forked_from_project.id}`, params: { license: true } }) license = templateProject.license?.name || templateProject.license?.nickname || templateProject.license?.key } catch (e) { // Template may be private or inaccessible } forkedFrom = { id: String(project.forked_from_project.id), name: project.forked_from_project.name, webUrl: project.forked_from_project.web_url, license } } const result = { websiteId, name: project.name.replace(this.options.repoPrefix, ''), imageUrl: project.avatar_url, createdAt: project.created_at, updatedAt: project.last_activity_at, connectorUserSettings: {}, visibility: project.visibility, repoUrl: project.web_url, forkCount: project.forks_count, starCount: project.star_count, forkedFrom, pagesUrl, pagesVisibility, lastJob } console.log('META', {result, websiteId}) return result } async setWebsiteMeta(session: GitlabSession, websiteId: WebsiteId, websiteMeta: WebsiteMetaFileContent): Promise { // Rename the repo if needed const oldMeta = await this.getWebsiteMeta(session, websiteId) if (websiteMeta.name !== oldMeta.name) { await this.callApi({ session, path: `api/v4/projects/${websiteId}`, method: 'PUT', requestBody: { name: this.options.repoPrefix + websiteMeta.name, } }) } } async writeAssets(session: GitlabSession, websiteId: string, files: ConnectorFile[], status?: StatusCallback, removeUnlisted = false): Promise { status && await status({ message: `Preparing ${files.length} files`, status: JobStatus.IN_PROGRESS }) // List all the files in assets folder const existingFiles = await this.ls({ session, websiteId, recursive: true, path: this.options.assetsFolder, }) // Create the actions for the batch const filesToUpload = [] as GitlabAction[] const filesToKeep = new Set(files.map(file => this.getAssetPath(file.path, false))) for (const file of files) { const filePath = this.getAssetPath(file.path, false) const content = (await contentToBuffer(file.content)).toString('base64') const existingSha = existingFiles.get(filePath) const newSha = computeGitBlobSha(content, true) if (existingSha) { if (existingSha !== newSha) { filesToUpload.push({ action: 'update', file_path: filePath, content, encoding: 'base64', }) } // else: skip unchanged file } else { filesToUpload.push({ action: 'create', file_path: filePath, content, encoding: 'base64', }) } } // Optionally remove unlisted files if (removeUnlisted) { for (const [existingFilePath] of existingFiles) { if (!filesToKeep.has(existingFilePath)) { filesToUpload.push({ action: 'delete', file_path: existingFilePath, }) } } } // Split the files into chunks to avoid the number of files limit const chunks: GitlabAction[][] = [] for (let i = 0; i < filesToUpload.length; i += MAX_BATCH_UPLOAD_SIZE) { chunks.push(filesToUpload.slice(i, i + MAX_BATCH_UPLOAD_SIZE)) } // Notify the user if nothing changed if (chunks.length === 0) { console.info('No files to upload') status && await status({ message: 'No files to upload', status: JobStatus.SUCCESS }) return } // Upload the files in chunks for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) { const chunk = chunks[chunkIndex] if (chunks.length > 1) { status && await status({ message: `Batch ${chunkIndex + 1}/${chunks.length}: Uploading ${chunk.length} files`, status: JobStatus.IN_PROGRESS }) } else { status && await status({ message: `Uploading ${files.length} file(s)`, status: JobStatus.IN_PROGRESS }) } try { await this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { branch: 'main', commit_message: `Batch update assets (${chunkIndex + 1}/${chunks.length})`, actions: chunk, } }) } catch (e) { console.error(`Batch ${chunkIndex + 1} failed`, e) status && await status({ message: `Error in batch ${chunkIndex + 1}`, status: JobStatus.ERROR }) throw e } } status && await status({ message: `All ${files.length} files uploaded`, status: JobStatus.SUCCESS }) } async readAsset(session: GitlabSession, websiteId: string, fileName: string): Promise { const finalPath = this.getAssetPath(fileName, false) return this.readFile(session, websiteId, finalPath) } async deleteAssets(session: GitlabSession, websiteId: string, fileNames: string[]): Promise { return this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/commits`, method: 'POST', requestBody: { id: websiteId, branch: this.options.branch, commit_message: `Delete assets from Silex: ${fileNames.join(', ')}`, actions: fileNames.map(f => ({ action: 'delete', file_path: this.getAssetPath(f), })), } }) } /* * Get the meta repo path for the current user * The meta repo contains a JSON file which contains the list of websites */ //private getMetaRepoPath(session: GitlabSession): string { // if(!this.getSessionToken(session).username) throw new ApiError('Missing Gitlab user ID. User not logged in?', 401) // return encodeURIComponent(`${this.getSessionToken(session).username}/${this.options.metaRepo}`) //} ///** // * Initialize the storage with a meta repo // */ //private async initStorage(session: GitlabSession): Promise { // // Create the meta repo // try { // const project = await this.callApi(session, 'api/v4/projects/', 'POST', { // name: this.options.metaRepo, // }) as any // return this.createFile(session, this.getMetaRepoPath(session), this.options.metaRepoFile, JSON.stringify({ // websites: {} // } as MetaRepoFileContent)) // } catch (e) { // console.error('Could not init storage', e.statusCode, e.httpStatusCode, e) // throw e // } //} /** * List all the files in a folder * The result is a map of file paths to their SHA */ protected async ls({ session, websiteId, recursive = false, path, }: { session: GitlabSession, websiteId: string, recursive?: boolean, path?: string, }): Promise> { const existingPaths = new Map() let page = 1 let keepGoing = true while (keepGoing) { const responseHeaders: any = {} let tree: any[] = [] try { const params = { recursive, per_page: 100, page, } as any if (path) params.path = path tree = await this.callApi({ session, path: `api/v4/projects/${websiteId}/repository/tree`, method: 'GET', params, responseHeaders, }) } catch (e) { // Allow 404 errors // This happens when the folder does not exist // In git this just means the files don't exist yet if (e.statusCode !== 404 && e.httpStatusCode !== 404) { throw e } } // Filter the files tree .filter(item => item.type === 'blob') .forEach(item => existingPaths.set(item.path, item.id)) // Check if we need to keep going const maxPages = responseHeaders['x-total-pages'] ? parseInt(responseHeaders['x-total-pages'], 10) : 1 keepGoing = page < maxPages page++ } // Return the set of existing paths return existingPaths } }