diff --git a/.github/actions/cluster-auth/action.yml b/.github/actions/cluster-auth/action.yml deleted file mode 100644 index 3fa3c61852..0000000000 --- a/.github/actions/cluster-auth/action.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: 'Auth to Cluster' -description: 'Auth to the GCP Cluster' -inputs: - gcp-project-number: - description: 'GCP project number' - required: true - gcp-project-id: - description: 'GCP project id' - required: true - service-account: - description: 'Service account' - cluster-name: - description: 'Cluster name' - cluster-location: - description: 'Cluster location' - -runs: - using: 'composite' - steps: - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: 'projects/${{ inputs.gcp-project-number }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy' - service_account: '${{ inputs.service-account }}' - token_format: 'access_token' - project_id: '${{ inputs.gcp-project-id }}' - - - name: 'Setup gcloud cli' - uses: 'google-github-actions/setup-gcloud@v2' - with: - install_components: 'gke-gcloud-auth-plugin' - - - id: get-gke-credentials - shell: bash - run: | - gcloud container clusters get-credentials ${{ inputs.cluster-name }} --region ${{ inputs.cluster-location }} --project ${{ inputs.gcp-project-id }} diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml deleted file mode 100644 index afa60601a4..0000000000 --- a/.github/actions/deploy/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: 'Deploy to Cluster' -description: 'Deploy AFFiNE Cloud to cluster' -inputs: - gcp-project-number: - description: 'GCP project number' - required: true - gcp-project-id: - description: 'GCP project id' - required: true - service-account: - description: 'Service account' - cluster-name: - description: 'Cluster name' - cluster-location: - description: 'Cluster location' - -runs: - using: 'composite' - steps: - - name: Setup Git short hash - shell: bash - run: | - echo "GIT_SHORT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - - name: 'Auth to cluster' - uses: './.github/actions/cluster-auth' - with: - gcp-project-number: '${{ inputs.gcp-project-number }}' - gcp-project-id: '${{ inputs.gcp-project-id }}' - service-account: '${{ inputs.service-account }}' - cluster-name: '${{ inputs.cluster-name }}' - cluster-location: '${{ inputs.cluster-location }}' - - - name: Deploy - shell: bash - run: node ./.github/actions/deploy/deploy.mjs diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs deleted file mode 100644 index c82782dfe6..0000000000 --- a/.github/actions/deploy/deploy.mjs +++ /dev/null @@ -1,173 +0,0 @@ -import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; - -const { - APP_VERSION, - BUILD_TYPE = 'canary', - GIT_SHORT_HASH, - SERVER_CONFIG, - GCLOUD_CONNECTION_NAME, - CLOUD_SQL_IAM_ACCOUNT, - APP_IAM_ACCOUNT, - STATIC_IP_NAME, -} = process.env; - -const namespaces = { - stable: 'production', - beta: 'beta', - canary: 'dev', - internal: 'internal', -}; -const namespace = namespaces[BUILD_TYPE]; -if (!namespace || !APP_VERSION || !GIT_SHORT_HASH) { - throw new Error( - 'BUILD_TYPE, APP_VERSION and GIT_SHORT_HASH must identify the release' - ); -} -if (!SERVER_CONFIG) { - throw new Error( - `Configure the ${BUILD_TYPE} environment SERVER_CONFIG secret before deploying` - ); -} -let serverConfig; -try { - serverConfig = JSON.parse(SERVER_CONFIG); -} catch { - throw new Error('SERVER_CONFIG must be valid JSON'); -} -if ( - serverConfig?.deployment?.type !== 'cloud' || - !serverConfig.db?.datasourceUrl || - !serverConfig.redis?.host || - !serverConfig.crypto?.privateKey || - !serverConfig.server?.host || - !['manticoresearch', 'elasticsearch'].includes( - serverConfig.indexer?.provider?.type - ) -) { - throw new Error( - 'SERVER_CONFIG requires cloud deployment, database, Redis, crypto key, server host and a remote search provider' - ); -} -const database = new URL(serverConfig.db.datasourceUrl); -const hosts = [ - ...new Set([serverConfig.server.host, ...(serverConfig.server.hosts ?? [])]), -]; -const cloudSql = ['stable', 'beta', 'internal'].includes(BUILD_TYPE); -const spot = ['beta', 'canary'].includes(BUILD_TYPE); -const imageTag = `${BUILD_TYPE}-${GIT_SHORT_HASH}`; -const serviceAccount = { - annotations: { 'iam.gke.io/gcp-service-account': APP_IAM_ACCOUNT }, -}; -const nodeSelector = spot ? { 'cloud.google.com/gke-spot': 'true' } : {}; -const neg = { 'cloud.google.com/neg': '{"ingress": true}' }; -const replicas = { - stable: { - front: Number(process.env.PRODUCTION_FRONT_REPLICA) || 2, - graphql: Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 2, - }, - beta: { - front: Number(process.env.BETA_FRONT_REPLICA) || 1, - graphql: Number(process.env.BETA_GRAPHQL_REPLICA) || 1, - }, -}; -const requests = { - beta: { - front: { cpu: '1', memory: '2Gi' }, - graphql: { cpu: '1', memory: '1Gi' }, - }, - canary: { - front: { cpu: '500m', memory: '512Mi' }, - graphql: { cpu: '1', memory: '512Mi' }, - }, -}; -const values = { - global: { - serverConfig, - app: { buildType: BUILD_TYPE }, - version: APP_VERSION, - database: { - gcloud: { enabled: cloudSql }, - host: database.hostname, - port: database.port || '5432', - }, - ingress: { - enabled: true, - hosts, - annotations: { - 'kubernetes.io/ingress.class': 'gce', - 'kubernetes.io/ingress.allow-http': 'true', - 'kubernetes.io/ingress.global-static-ip-name': STATIC_IP_NAME, - }, - }, - }, - front: { - image: { tag: imageTag }, - replicaCount: replicas[BUILD_TYPE]?.front ?? 1, - serviceAccount, - nodeSelector, - resources: { requests: requests[BUILD_TYPE]?.front ?? {} }, - ...(cloudSql - ? { - services: Object.fromEntries( - ['web', 'sync', 'renderer'].map(name => [ - name, - { annotations: neg }, - ]) - ), - } - : {}), - }, - graphql: { - image: { tag: imageTag }, - replicaCount: replicas[BUILD_TYPE]?.graphql ?? 1, - serviceAccount, - nodeSelector, - resources: { requests: requests[BUILD_TYPE]?.graphql ?? {} }, - ...(cloudSql ? { service: { annotations: neg } } : {}), - }, - worker: { image: { tag: imageTag } }, - ...(cloudSql - ? { - 'cloud-sql-proxy': { - database: { connectionName: GCLOUD_CONNECTION_NAME }, - serviceAccount: { - annotations: { - 'iam.gke.io/gcp-service-account': CLOUD_SQL_IAM_ACCOUNT, - }, - }, - nodeSelector: { - 'iam.gke.io/gke-metadata-server-enabled': 'true', - ...(BUILD_TYPE === 'beta' ? nodeSelector : {}), - }, - }, - } - : {}), -}; - -const directory = mkdtempSync(join(tmpdir(), 'affine-deploy-')); -try { - const valuesFile = join(directory, 'values.json'); - writeFileSync(valuesFile, JSON.stringify(values), { mode: 0o600 }); - const args = [ - 'upgrade', - '--install', - 'affine', - '.github/helm/affine', - '--namespace', - namespace, - '--values', - valuesFile, - '--timeout', - '10m', - ]; - // Helm dry-run output includes Secrets; validate it without logging the manifests. - execFileSync('helm', [...args, '--dry-run', '--hide-secret'], { - stdio: ['inherit', 'ignore', 'inherit'], - }); - execFileSync('helm', [...args, '--atomic'], { stdio: 'inherit' }); -} finally { - rmSync(directory, { recursive: true, force: true }); -} diff --git a/.github/actions/deploy/deploy.spec.mjs b/.github/actions/deploy/deploy.spec.mjs deleted file mode 100644 index e955d59524..0000000000 --- a/.github/actions/deploy/deploy.spec.mjs +++ /dev/null @@ -1,150 +0,0 @@ -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { - existsSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { tmpdir } from 'node:os'; -import { delimiter, join, resolve } from 'node:path'; -import { snapshot, test } from 'node:test'; - -snapshot.setDefaultSnapshotSerializers([ - value => (typeof value === 'string' ? value : JSON.stringify(value, null, 2)), -]); - -const config = { - deployment: { type: 'cloud' }, - db: { datasourceUrl: 'postgresql://user:p%40ss@sql-proxy:5432/affine' }, - redis: { host: 'redis', password: 'comma,quote"dollar$backtick`' }, - crypto: { privateKey: 'test-key\nsecond-line' }, - server: { - host: 'affine.example', - hosts: ['affine.example', 'alias.example'], - https: true, - path: '', - }, - indexer: { - provider: { - type: 'elasticsearch', - endpoint: 'https://search.example', - apiKey: 'test-key', - }, - }, - payment: { stripe: { accountId: 'acct_test', environment: 'test' } }, -}; - -for (const [channel, namespace] of Object.entries({ - stable: 'production', - beta: 'beta', - canary: 'dev', - internal: 'internal', -})) { - test(`${channel}: config reaches every role and migration without shell interpolation`, t => { - const directory = mkdtempSync(join(tmpdir(), 'affine-deploy-test-')); - try { - writeFileSync( - join(directory, 'helm'), - `#!/usr/bin/env node -const fs = require('node:fs'); -const args = process.argv.slice(2); -const file = args[args.indexOf('--values') + 1]; -fs.appendFileSync(process.env.DEPLOY_TEST_LOG, JSON.stringify({args, mode: fs.statSync(file).mode & 0o777, values: JSON.parse(fs.readFileSync(file, 'utf8'))}) + '\\n'); -`, - { mode: 0o700 } - ); - const log = join(directory, 'calls.jsonl'); - const result = spawnSync( - process.execPath, - ['.github/actions/deploy/deploy.mjs'], - { - encoding: 'utf8', - env: { - ...process.env, - PATH: directory + delimiter + process.env.PATH, - DEPLOY_TEST_LOG: log, - BUILD_TYPE: channel, - APP_VERSION: '0.27.5-beta.2', - GIT_SHORT_HASH: 'abcdef', - SERVER_CONFIG: JSON.stringify(config), - GCLOUD_CONNECTION_NAME: 'project:region:database', - CLOUD_SQL_IAM_ACCOUNT: 'sql@project.iam.gserviceaccount.com', - APP_IAM_ACCOUNT: 'app@project.iam.gserviceaccount.com', - STATIC_IP_NAME: 'affine-ip', - PRODUCTION_FRONT_REPLICA: '2', - PRODUCTION_GRAPHQL_REPLICA: '2', - BETA_FRONT_REPLICA: '1', - BETA_GRAPHQL_REPLICA: '1', - }, - } - ); - assert.equal(result.status, 0, result.stderr); - const calls = readFileSync(log, 'utf8') - .trim() - .split('\n') - .map(line => JSON.parse(line)); - assert.equal(calls[0].mode, 0o600); - assert.equal( - existsSync(calls[0].args[calls[0].args.indexOf('--values') + 1]), - false - ); - const valuesFile = join(directory, 'values.json'); - writeFileSync(valuesFile, JSON.stringify(calls[0].values)); - const rendered = spawnSync( - 'helm', - [ - 'template', - 'affine', - '.github/helm/affine', - '--namespace', - namespace, - '--values', - valuesFile, - ], - { encoding: 'utf8' } - ); - assert.equal(rendered.status, 0, rendered.stderr); - t.assert.snapshot({ - commands: calls.map(call => - call.args.map(arg => - arg === call.args[call.args.indexOf('--values') + 1] - ? '' - : arg - ) - ), - values: calls[0].values, - }); - if (channel === 'beta') t.assert.snapshot(rendered.stdout); - assert.ok(!result.stdout.includes(config.crypto.privateKey)); - } finally { - rmSync(directory, { recursive: true, force: true }); - } - }); -} - -for (const invalid of [ - '', - '{invalid-json', - JSON.stringify({ ...config, deployment: { type: 'selfhosted' } }), -]) { - test(`invalid cloud config is rejected before Helm: ${invalid.slice(0, 20)}`, () => { - const result = spawnSync( - process.execPath, - [resolve('.github/actions/deploy/deploy.mjs')], - { - encoding: 'utf8', - env: { - ...process.env, - BUILD_TYPE: 'beta', - APP_VERSION: '0.27.5-beta.2', - GIT_SHORT_HASH: 'abcdef', - SERVER_CONFIG: invalid, - }, - } - ); - assert.notEqual(result.status, 0); - assert.match(result.stderr, /SERVER_CONFIG/); - }); -} diff --git a/.github/actions/deploy/deploy.spec.mjs.snapshot b/.github/actions/deploy/deploy.spec.mjs.snapshot deleted file mode 100644 index c23d015677..0000000000 --- a/.github/actions/deploy/deploy.spec.mjs.snapshot +++ /dev/null @@ -1,1653 +0,0 @@ -exports[`beta: config reaches every role and migration without shell interpolation 1`] = ` -{ - "commands": [ - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "beta", - "--values", - "", - "--timeout", - "10m", - "--dry-run", - "--hide-secret" - ], - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "beta", - "--values", - "", - "--timeout", - "10m", - "--atomic" - ] - ], - "values": { - "global": { - "serverConfig": { - "deployment": { - "type": "cloud" - }, - "db": { - "datasourceUrl": "postgresql://user:p%40ss@sql-proxy:5432/affine" - }, - "redis": { - "host": "redis", - "password": "comma,quote\\"dollar$backtick\`" - }, - "crypto": { - "privateKey": "test-key\\nsecond-line" - }, - "server": { - "host": "affine.example", - "hosts": [ - "affine.example", - "alias.example" - ], - "https": true, - "path": "" - }, - "indexer": { - "provider": { - "type": "elasticsearch", - "endpoint": "https://search.example", - "apiKey": "test-key" - } - }, - "payment": { - "stripe": { - "accountId": "acct_test", - "environment": "test" - } - } - }, - "app": { - "buildType": "beta" - }, - "version": "0.27.5-beta.2", - "database": { - "gcloud": { - "enabled": true - }, - "host": "sql-proxy", - "port": "5432" - }, - "ingress": { - "enabled": true, - "hosts": [ - "affine.example", - "alias.example" - ], - "annotations": { - "kubernetes.io/ingress.class": "gce", - "kubernetes.io/ingress.allow-http": "true", - "kubernetes.io/ingress.global-static-ip-name": "affine-ip" - } - } - }, - "front": { - "image": { - "tag": "beta-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "cloud.google.com/gke-spot": "true" - }, - "resources": { - "requests": { - "cpu": "1", - "memory": "2Gi" - } - }, - "services": { - "web": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "sync": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "renderer": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - } - }, - "graphql": { - "image": { - "tag": "beta-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "cloud.google.com/gke-spot": "true" - }, - "resources": { - "requests": { - "cpu": "1", - "memory": "1Gi" - } - }, - "service": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - }, - "worker": { - "image": { - "tag": "beta-abcdef" - } - }, - "cloud-sql-proxy": { - "database": { - "connectionName": "project:region:database" - }, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "sql@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "iam.gke.io/gke-metadata-server-enabled": "true", - "cloud.google.com/gke-spot": "true" - } - } - } -} -`; - -exports[`beta: config reaches every role and migration without shell interpolation 2`] = ` ---- -# Source: affine/charts/cloud-sql-proxy/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: affine-cloud-sql-proxy - labels: - helm.sh/chart: cloud-sql-proxy-0.0.0 - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "2.26.2" - app.kubernetes.io/managed-by: Helm - annotations: - iam.gke.io/gcp-service-account: sql@project.iam.gserviceaccount.com -automountServiceAccountToken: true - ---- -# Source: affine/charts/front/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: affine-front - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - iam.gke.io/gcp-service-account: app@project.iam.gserviceaccount.com - ---- -# Source: affine/charts/graphql/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: affine-graphql - labels: - helm.sh/chart: graphql-0.0.0 - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - iam.gke.io/gcp-service-account: app@project.iam.gserviceaccount.com - ---- -# Source: affine/templates/worker-serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: affine-worker - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: affine - app.kubernetes.io/managed-by: Helm - ---- -# Source: affine/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: affine-runtime-config -data: - web-assets-manifest: |- - - - mobile-assets-manifest: |- - - - ---- -# Source: affine/charts/cloud-sql-proxy/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: affine-cloud-sql-proxy - labels: - helm.sh/chart: cloud-sql-proxy-0.0.0 - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "2.26.2" - app.kubernetes.io/managed-by: Helm -spec: - type: ClusterIP - ports: - - port: 5432 - targetPort: cloud-sql-proxy - protocol: TCP - name: cloud-sql-proxy - - port: 9801 - targetPort: 9801 - protocol: TCP - name: health - selector: - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - ---- -# Source: affine/charts/front/templates/service-renderer.yaml -apiVersion: v1 -kind: Service -metadata: - name: affine-renderer - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' - cloud.google.com/neg: '{"ingress": true}' -spec: - type: ClusterIP - ports: - - port: 3000 - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - ---- -# Source: affine/charts/front/templates/service-sync.yaml -apiVersion: v1 -kind: Service -metadata: - name: affine-sync - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - cloud.google.com/backend-config: "{\\"default\\":\\"affine-sync-backendconfig\\"}" - cloud.google.com/neg: '{"ingress": true}' -spec: - type: ClusterIP - ports: - - port: 3010 - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - ---- -# Source: affine/charts/front/templates/service-web.yaml -apiVersion: v1 -kind: Service -metadata: - name: affine-web - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - cloud.google.com/neg: '{"ingress": true}' -spec: - type: ClusterIP - ports: - - port: 8080 - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - ---- -# Source: affine/charts/graphql/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: affine-graphql - labels: - helm.sh/chart: graphql-0.0.0 - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' - cloud.google.com/neg: '{"ingress": true}' -spec: - type: ClusterIP - ports: - - port: 3000 - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - ---- -# Source: affine/charts/cloud-sql-proxy/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: affine-cloud-sql-proxy - labels: - helm.sh/chart: cloud-sql-proxy-0.0.0 - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "2.26.2" - app.kubernetes.io/managed-by: Helm -spec: - replicas: 2 - selector: - matchLabels: - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - template: - metadata: - labels: - helm.sh/chart: cloud-sql-proxy-0.0.0 - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "2.26.2" - app.kubernetes.io/managed-by: Helm - spec: - serviceAccountName: affine-cloud-sql-proxy - securityContext: - fsGroup: 2000 - seccompProfile: - type: RuntimeDefault - containers: - - name: cloud-sql-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - image: "ghcr.io/toeverything/affine:cloud-sql-proxy-2.26.2" - imagePullPolicy: IfNotPresent - args: - - "--address" - - "0.0.0.0" - - "--structured-logs" - - "--auto-iam-authn" - - "project:region:database" - env: - # Enable HTTP healthchecks on port 9801. This enables /liveness, - # /readiness and /startup health check endpoints. Allow connections - # listen for connections on any interface (0.0.0.0) so that the - # k8s management components can reach these endpoints. - - name: CSQL_PROXY_HEALTH_CHECK - value: "true" - - name: CSQL_PROXY_HTTP_PORT - value: "9801" - - name: CSQL_PROXY_HTTP_ADDRESS - value: 0.0.0.0 - ports: - - name: cloud-sql-proxy - containerPort: 5432 - protocol: TCP - - containerPort: 9801 - protocol: TCP - # The /startup probe returns OK when the proxy is ready to receive - # connections from the application. In this example, k8s will check - # once a second for 60 seconds. - startupProbe: - failureThreshold: 60 - httpGet: - path: /startup - port: 9801 - scheme: HTTP - periodSeconds: 1 - successThreshold: 1 - timeoutSeconds: 10 - # The /liveness probe returns OK as soon as the proxy application has - # begun its startup process and continues to return OK until the - # process stops. - livenessProbe: - failureThreshold: 3 - httpGet: - path: /liveness - port: 9801 - scheme: HTTP - # The probe will be checked every 10 seconds. - periodSeconds: 10 - # Number of times the probe is allowed to fail before the transition - # from healthy to failure state. - # - # If periodSeconds = 60, 5 tries will result in five minutes of - # checks. The proxy starts to refresh a certificate five minutes - # before its expiration. If those five minutes lapse without a - # successful refresh, the liveness probe will fail and the pod will be - # restarted. - successThreshold: 1 - # The probe will fail if it does not respond in 10 seconds - timeoutSeconds: 10 - readinessProbe: - # The /readiness probe returns OK when the proxy can establish - # a new connections to its databases. - httpGet: - path: /readiness - port: 9801 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 10 - # Number of times the probe must report success to transition from failure to healthy state. - # Defaults to 1 for readiness probe. - successThreshold: 1 - failureThreshold: 6 - resources: - limits: - cpu: "1" - memory: 1Gi - requests: - cpu: 100m - memory: 512Mi - nodeSelector: - cloud.google.com/gke-spot: "true" - iam.gke.io/gke-metadata-server-enabled: "true" - ---- -# Source: affine/charts/front/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: affine-front - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - template: - metadata: - annotations: - checksum/server-config: 0bc60d6fccd7467555d683a9101f844b951c511c428cbdd24416b9af8c241cce - labels: - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - spec: - serviceAccountName: affine-front - securityContext: - fsGroup: 2000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://sql-proxy:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - containers: - - name: front - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - image: "ghcr.io/toeverything/affine:beta-abcdef" - imagePullPolicy: IfNotPresent - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "production" - - name: NODE_OPTIONS - value: "--max-old-space-size=3072" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "front" - - name: AFFINE_ENV - value: "beta" - ports: - - name: http - containerPort: 3010 - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 60 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 12 - successThreshold: 1 - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - resources: - limits: - cpu: "1" - memory: 6Gi - requests: - cpu: "1" - memory: 2Gi - volumeMounts: - - name: tmp - mountPath: /tmp - - name: server-config - mountPath: /app/config.json - subPath: front.json - readOnly: true - volumes: - - name: tmp - emptyDir: {} - - name: server-config - secret: - secretName: server-config - nodeSelector: - cloud.google.com/gke-spot: "true" - ---- -# Source: affine/charts/graphql/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: affine-graphql - labels: - helm.sh/chart: graphql-0.0.0 - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - template: - metadata: - annotations: - checksum/server-config: 4173541b74d5d94560959cf10691d31af23af149a72856732c292a65fe606eaf - labels: - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - spec: - serviceAccountName: affine-graphql - securityContext: - fsGroup: 2000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://sql-proxy:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - containers: - - name: graphql - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - image: "ghcr.io/toeverything/affine:beta-abcdef" - imagePullPolicy: IfNotPresent - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "production" - - name: NODE_OPTIONS - value: "--max-old-space-size=2048" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "graphql" - - name: AFFINE_ENV - value: "beta" - ports: - - name: http - containerPort: 3000 - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 12 - successThreshold: 1 - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - resources: - limits: - cpu: "1" - memory: 4Gi - requests: - cpu: "1" - memory: 1Gi - volumeMounts: - - name: tmp - mountPath: /tmp - - name: server-config - mountPath: /app/config.json - subPath: graphql.json - readOnly: true - volumes: - - name: tmp - emptyDir: {} - - name: server-config - secret: - secretName: server-config - nodeSelector: - cloud.google.com/gke-spot: "true" - ---- -# Source: affine/templates/worker-deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: affine-worker - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: affine - template: - metadata: - annotations: - checksum/server-config: 8238095e3789fcc0db0c064082e3c4963ea003bf8830b43cd16f73e90aeca195 - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: affine - spec: - serviceAccountName: affine-worker - securityContext: - fsGroup: 2000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://sql-proxy:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - containers: - - name: worker - image: "ghcr.io/toeverything/affine:beta-abcdef" - imagePullPolicy: IfNotPresent - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "production" - - name: NODE_OPTIONS - value: "--max-old-space-size=3072" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "worker" - - name: AFFINE_ENV - value: "beta" - ports: - - name: http - containerPort: 3002 - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 60 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 12 - successThreshold: 1 - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - resources: - limits: - cpu: "1" - memory: 4Gi - requests: - cpu: "1" - memory: 2Gi - volumeMounts: - - name: server-config - mountPath: /app/config.json - subPath: worker.json - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: server-config - secret: - secretName: server-config - - name: tmp - emptyDir: {} - ---- -# Source: affine/templates/ingress.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: affine - labels: - helm.sh/chart: affine-0.0.0 - app.kubernetes.io/name: affine - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - annotations: - kubernetes.io/ingress.allow-http: "true" - kubernetes.io/ingress.class: gce - kubernetes.io/ingress.global-static-ip-name: affine-ip -spec: - rules: - - host: "affine.example" - http: - paths: - - path: /socket.io - pathType: Prefix - backend: - service: - name: affine-sync - port: - number: 3010 - - path: /graphql - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: 3000 - - path: /api - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: 3000 - - path: /workspace - pathType: Prefix - backend: - service: - name: affine-renderer - port: - number: 3000 - - path: / - pathType: Prefix - backend: - service: - name: affine-web - port: - number: 8080 - - host: "alias.example" - http: - paths: - - path: /socket.io - pathType: Prefix - backend: - service: - name: affine-sync - port: - number: 3010 - - path: /graphql - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: 3000 - - path: /api - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: 3000 - - path: /workspace - pathType: Prefix - backend: - service: - name: affine-renderer - port: - number: 3000 - - path: / - pathType: Prefix - backend: - service: - name: affine-web - port: - number: 8080 - ---- -# Source: affine/templates/backend-config.yaml -apiVersion: cloud.google.com/v1 -kind: BackendConfig -metadata: - name: affine-api-backendconfig - annotations: - meta.helm.sh/release-name: affine - meta.helm.sh/release-namespace: beta - labels: - helm.sh/chart: affine-0.0.0 - app.kubernetes.io/name: affine - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm -spec: - healthCheck: - timeoutSec: 1 - type: HTTP - requestPath: /info ---- -# Source: affine/templates/backend-config.yaml -apiVersion: cloud.google.com/v1 -kind: BackendConfig -metadata: - name: affine-sync-backendconfig - annotations: - meta.helm.sh/release-name: affine - meta.helm.sh/release-namespace: beta - labels: - helm.sh/chart: affine-0.0.0 - app.kubernetes.io/name: affine - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm -spec: - timeoutSec: 10800 - healthCheck: - timeoutSec: 1 - type: HTTP - requestPath: /info - ---- -# Source: affine/templates/monitoring.yaml -apiVersion: monitoring.googleapis.com/v1 -kind: PodMonitoring -metadata: - name: "affine-monitoring" -spec: - selector: - matchLabels: - app.kubernetes.io/instance: affine - endpoints: - - port: 9464 - interval: 30s ---- -# Source: affine/templates/server-config-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: server-config - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-2" - "helm.sh/hook-delete-policy": before-hook-creation -type: Opaque -data: - front.json: "eyJjcnlwdG8iOnsicHJpdmF0ZUtleSI6InRlc3Qta2V5XG5zZWNvbmQtbGluZSJ9LCJkYiI6eyJkYXRhc291cmNlVXJsIjoicG9zdGdyZXNxbDovL3VzZXI6cCU0MHNzQHNxbC1wcm94eTo1NDMyL2FmZmluZSJ9LCJkZXBsb3ltZW50Ijp7InR5cGUiOiJjbG91ZCJ9LCJpbmRleGVyIjp7InByb3ZpZGVyIjp7ImFwaUtleSI6InRlc3Qta2V5IiwiZW5kcG9pbnQiOiJodHRwczovL3NlYXJjaC5leGFtcGxlIiwidHlwZSI6ImVsYXN0aWNzZWFyY2gifX0sInBheW1lbnQiOnsic3RyaXBlIjp7ImFjY291bnRJZCI6ImFjY3RfdGVzdCIsImVudmlyb25tZW50IjoidGVzdCJ9fSwicmVkaXMiOnsiaG9zdCI6InJlZGlzIiwicGFzc3dvcmQiOiJjb21tYSxxdW90ZVwiZG9sbGFyJGJhY2t0aWNrYCJ9LCJzZXJ2ZXIiOnsiaG9zdCI6ImFmZmluZS5leGFtcGxlIiwiaG9zdHMiOlsiYWZmaW5lLmV4YW1wbGUiLCJhbGlhcy5leGFtcGxlIl0sImh0dHBzIjp0cnVlLCJwYXRoIjoiIiwicG9ydCI6MzAxMH19" - graphql.json: "eyJjcnlwdG8iOnsicHJpdmF0ZUtleSI6InRlc3Qta2V5XG5zZWNvbmQtbGluZSJ9LCJkYiI6eyJkYXRhc291cmNlVXJsIjoicG9zdGdyZXNxbDovL3VzZXI6cCU0MHNzQHNxbC1wcm94eTo1NDMyL2FmZmluZSJ9LCJkZXBsb3ltZW50Ijp7InR5cGUiOiJjbG91ZCJ9LCJpbmRleGVyIjp7InByb3ZpZGVyIjp7ImFwaUtleSI6InRlc3Qta2V5IiwiZW5kcG9pbnQiOiJodHRwczovL3NlYXJjaC5leGFtcGxlIiwidHlwZSI6ImVsYXN0aWNzZWFyY2gifX0sInBheW1lbnQiOnsic3RyaXBlIjp7ImFjY291bnRJZCI6ImFjY3RfdGVzdCIsImVudmlyb25tZW50IjoidGVzdCJ9fSwicmVkaXMiOnsiaG9zdCI6InJlZGlzIiwicGFzc3dvcmQiOiJjb21tYSxxdW90ZVwiZG9sbGFyJGJhY2t0aWNrYCJ9LCJzZXJ2ZXIiOnsiaG9zdCI6ImFmZmluZS5leGFtcGxlIiwiaG9zdHMiOlsiYWZmaW5lLmV4YW1wbGUiLCJhbGlhcy5leGFtcGxlIl0sImh0dHBzIjp0cnVlLCJwYXRoIjoiIiwicG9ydCI6MzAwMH19" - worker.json: "eyJjcnlwdG8iOnsicHJpdmF0ZUtleSI6InRlc3Qta2V5XG5zZWNvbmQtbGluZSJ9LCJkYiI6eyJkYXRhc291cmNlVXJsIjoicG9zdGdyZXNxbDovL3VzZXI6cCU0MHNzQHNxbC1wcm94eTo1NDMyL2FmZmluZSJ9LCJkZXBsb3ltZW50Ijp7InR5cGUiOiJjbG91ZCJ9LCJpbmRleGVyIjp7InByb3ZpZGVyIjp7ImFwaUtleSI6InRlc3Qta2V5IiwiZW5kcG9pbnQiOiJodHRwczovL3NlYXJjaC5leGFtcGxlIiwidHlwZSI6ImVsYXN0aWNzZWFyY2gifX0sInBheW1lbnQiOnsic3RyaXBlIjp7ImFjY291bnRJZCI6ImFjY3RfdGVzdCIsImVudmlyb25tZW50IjoidGVzdCJ9fSwicmVkaXMiOnsiaG9zdCI6InJlZGlzIiwicGFzc3dvcmQiOiJjb21tYSxxdW90ZVwiZG9sbGFyJGJhY2t0aWNrYCJ9LCJzZXJ2ZXIiOnsiaG9zdCI6ImFmZmluZS5leGFtcGxlIiwiaG9zdHMiOlsiYWZmaW5lLmV4YW1wbGUiLCJhbGlhcy5leGFtcGxlIl0sImh0dHBzIjp0cnVlLCJwYXRoIjoiIiwicG9ydCI6MzAwMn19" - database-url: "cG9zdGdyZXNxbDovL3VzZXI6cCU0MHNzQHNxbC1wcm94eTo1NDMyL2FmZmluZQ==" - ---- -# Source: affine/charts/cloud-sql-proxy/templates/tests/test-connection.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "affine-cloud-sql-proxy-test-connection" - labels: - helm.sh/chart: cloud-sql-proxy-0.0.0 - app.kubernetes.io/name: cloud-sql-proxy - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "2.26.2" - app.kubernetes.io/managed-by: Helm - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['affine-cloud-sql-proxy:5432'] - restartPolicy: Never - ---- -# Source: affine/charts/front/templates/tests/test-connection.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "affine-front-test-connection" - labels: - helm.sh/chart: front-0.0.0 - app.kubernetes.io/name: front - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['affine-sync:3010'] - restartPolicy: Never - ---- -# Source: affine/charts/graphql/templates/tests/test-connection.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "affine-graphql-test-connection" - labels: - helm.sh/chart: graphql-0.0.0 - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['affine-graphql:3000'] - restartPolicy: Never - ---- -# Source: affine/charts/graphql/templates/migration.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: affine-graphql-database-migration - labels: - helm.sh/chart: graphql-0.0.0 - app.kubernetes.io/name: graphql - app.kubernetes.io/instance: affine - app.kubernetes.io/version: "0.27.5" - app.kubernetes.io/managed-by: Helm - monitoring: enabled - annotations: - "helm.sh/hook": post-install,pre-upgrade - "helm.sh/hook-weight": "-1" - "helm.sh/hook-delete-policy": before-hook-creation - -spec: - template: - spec: - serviceAccountName: affine-graphql - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://sql-proxy:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - containers: - - name: graphql - image: "ghcr.io/toeverything/affine:beta-abcdef" - command: ["yarn", "predeploy"] - env: - # Prisma CLI still reads DATABASE_URL; derive it from the same config. - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "production" - - name: AFFINE_ENV - value: "beta" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - volumeMounts: - - name: server-config - mountPath: /app/config.json - subPath: graphql.json - readOnly: true - resources: - requests: - cpu: '100m' - memory: '200Mi' - volumes: - - name: server-config - secret: - secretName: server-config - restartPolicy: Never - backoffLimit: 1 - - -`; - -exports[`canary: config reaches every role and migration without shell interpolation 1`] = ` -{ - "commands": [ - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "dev", - "--values", - "", - "--timeout", - "10m", - "--dry-run", - "--hide-secret" - ], - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "dev", - "--values", - "", - "--timeout", - "10m", - "--atomic" - ] - ], - "values": { - "global": { - "serverConfig": { - "deployment": { - "type": "cloud" - }, - "db": { - "datasourceUrl": "postgresql://user:p%40ss@sql-proxy:5432/affine" - }, - "redis": { - "host": "redis", - "password": "comma,quote\\"dollar$backtick\`" - }, - "crypto": { - "privateKey": "test-key\\nsecond-line" - }, - "server": { - "host": "affine.example", - "hosts": [ - "affine.example", - "alias.example" - ], - "https": true, - "path": "" - }, - "indexer": { - "provider": { - "type": "elasticsearch", - "endpoint": "https://search.example", - "apiKey": "test-key" - } - }, - "payment": { - "stripe": { - "accountId": "acct_test", - "environment": "test" - } - } - }, - "app": { - "buildType": "canary" - }, - "version": "0.27.5-beta.2", - "database": { - "gcloud": { - "enabled": false - }, - "host": "sql-proxy", - "port": "5432" - }, - "ingress": { - "enabled": true, - "hosts": [ - "affine.example", - "alias.example" - ], - "annotations": { - "kubernetes.io/ingress.class": "gce", - "kubernetes.io/ingress.allow-http": "true", - "kubernetes.io/ingress.global-static-ip-name": "affine-ip" - } - } - }, - "front": { - "image": { - "tag": "canary-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "cloud.google.com/gke-spot": "true" - }, - "resources": { - "requests": { - "cpu": "500m", - "memory": "512Mi" - } - } - }, - "graphql": { - "image": { - "tag": "canary-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "cloud.google.com/gke-spot": "true" - }, - "resources": { - "requests": { - "cpu": "1", - "memory": "512Mi" - } - } - }, - "worker": { - "image": { - "tag": "canary-abcdef" - } - } - } -} -`; - -exports[`internal: config reaches every role and migration without shell interpolation 1`] = ` -{ - "commands": [ - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "internal", - "--values", - "", - "--timeout", - "10m", - "--dry-run", - "--hide-secret" - ], - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "internal", - "--values", - "", - "--timeout", - "10m", - "--atomic" - ] - ], - "values": { - "global": { - "serverConfig": { - "deployment": { - "type": "cloud" - }, - "db": { - "datasourceUrl": "postgresql://user:p%40ss@sql-proxy:5432/affine" - }, - "redis": { - "host": "redis", - "password": "comma,quote\\"dollar$backtick\`" - }, - "crypto": { - "privateKey": "test-key\\nsecond-line" - }, - "server": { - "host": "affine.example", - "hosts": [ - "affine.example", - "alias.example" - ], - "https": true, - "path": "" - }, - "indexer": { - "provider": { - "type": "elasticsearch", - "endpoint": "https://search.example", - "apiKey": "test-key" - } - }, - "payment": { - "stripe": { - "accountId": "acct_test", - "environment": "test" - } - } - }, - "app": { - "buildType": "internal" - }, - "version": "0.27.5-beta.2", - "database": { - "gcloud": { - "enabled": true - }, - "host": "sql-proxy", - "port": "5432" - }, - "ingress": { - "enabled": true, - "hosts": [ - "affine.example", - "alias.example" - ], - "annotations": { - "kubernetes.io/ingress.class": "gce", - "kubernetes.io/ingress.allow-http": "true", - "kubernetes.io/ingress.global-static-ip-name": "affine-ip" - } - } - }, - "front": { - "image": { - "tag": "internal-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": {}, - "resources": { - "requests": {} - }, - "services": { - "web": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "sync": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "renderer": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - } - }, - "graphql": { - "image": { - "tag": "internal-abcdef" - }, - "replicaCount": 1, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": {}, - "resources": { - "requests": {} - }, - "service": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - }, - "worker": { - "image": { - "tag": "internal-abcdef" - } - }, - "cloud-sql-proxy": { - "database": { - "connectionName": "project:region:database" - }, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "sql@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "iam.gke.io/gke-metadata-server-enabled": "true" - } - } - } -} -`; - -exports[`stable: config reaches every role and migration without shell interpolation 1`] = ` -{ - "commands": [ - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "production", - "--values", - "", - "--timeout", - "10m", - "--dry-run", - "--hide-secret" - ], - [ - "upgrade", - "--install", - "affine", - ".github/helm/affine", - "--namespace", - "production", - "--values", - "", - "--timeout", - "10m", - "--atomic" - ] - ], - "values": { - "global": { - "serverConfig": { - "deployment": { - "type": "cloud" - }, - "db": { - "datasourceUrl": "postgresql://user:p%40ss@sql-proxy:5432/affine" - }, - "redis": { - "host": "redis", - "password": "comma,quote\\"dollar$backtick\`" - }, - "crypto": { - "privateKey": "test-key\\nsecond-line" - }, - "server": { - "host": "affine.example", - "hosts": [ - "affine.example", - "alias.example" - ], - "https": true, - "path": "" - }, - "indexer": { - "provider": { - "type": "elasticsearch", - "endpoint": "https://search.example", - "apiKey": "test-key" - } - }, - "payment": { - "stripe": { - "accountId": "acct_test", - "environment": "test" - } - } - }, - "app": { - "buildType": "stable" - }, - "version": "0.27.5-beta.2", - "database": { - "gcloud": { - "enabled": true - }, - "host": "sql-proxy", - "port": "5432" - }, - "ingress": { - "enabled": true, - "hosts": [ - "affine.example", - "alias.example" - ], - "annotations": { - "kubernetes.io/ingress.class": "gce", - "kubernetes.io/ingress.allow-http": "true", - "kubernetes.io/ingress.global-static-ip-name": "affine-ip" - } - } - }, - "front": { - "image": { - "tag": "stable-abcdef" - }, - "replicaCount": 2, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": {}, - "resources": { - "requests": {} - }, - "services": { - "web": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "sync": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - }, - "renderer": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - } - }, - "graphql": { - "image": { - "tag": "stable-abcdef" - }, - "replicaCount": 2, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "app@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": {}, - "resources": { - "requests": {} - }, - "service": { - "annotations": { - "cloud.google.com/neg": "{\\"ingress\\": true}" - } - } - }, - "worker": { - "image": { - "tag": "stable-abcdef" - } - }, - "cloud-sql-proxy": { - "database": { - "connectionName": "project:region:database" - }, - "serviceAccount": { - "annotations": { - "iam.gke.io/gcp-service-account": "sql@project.iam.gserviceaccount.com" - } - }, - "nodeSelector": { - "iam.gke.io/gke-metadata-server-enabled": "true" - } - } - } -} -`; diff --git a/.github/actions/prepare-release/action.yml b/.github/actions/prepare-release/action.yml index 3eac38c167..d29b35453d 100644 --- a/.github/actions/prepare-release/action.yml +++ b/.github/actions/prepare-release/action.yml @@ -1,41 +1,70 @@ name: Prepare Release -description: 'Prepare Release' +description: Resolve the application release with the published affine-release tool +inputs: + targets: + description: Targets accepted by affine-release plan + required: true + created-at: + description: Original GitHub workflow run creation time + required: true + tool-image: + description: Release tool image tag or digest + required: true outputs: APP_VERSION: - description: 'App Version' - value: ${{ steps.get-version.outputs.APP_VERSION }} + description: Planned application version + value: ${{ steps.plan.outputs.APP_VERSION }} GIT_SHORT_HASH: - description: 'Git Short Hash' - value: ${{ steps.get-version.outputs.GIT_SHORT_HASH }} + description: Application source short SHA + value: ${{ steps.plan.outputs.GIT_SHORT_HASH }} BUILD_TYPE: - description: 'Build Type' - value: ${{ steps.get-version.outputs.BUILD_TYPE }} + description: Planned channel + value: ${{ steps.plan.outputs.BUILD_TYPE }} + SKIP: + description: Whether this source and target set is already complete + value: ${{ steps.plan.outputs.SKIP }} + SERVER: + description: Whether the server is selected + value: ${{ steps.plan.outputs.SERVER }} + DESKTOP_MATRIX: + description: Desktop build matrix + value: ${{ steps.plan.outputs.DESKTOP_MATRIX }} + MOBILE_MATRIX: + description: Mobile build matrix + value: ${{ steps.plan.outputs.MOBILE_MATRIX }} runs: - using: 'composite' + using: composite steps: - - name: Get Version - id: get-version + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Pull release tool shell: bash + env: + TOOL_IMAGE: ${{ inputs.tool-image }} + run: docker pull "$TOOL_IMAGE" + - name: Resolve release plan + id: plan + shell: bash + env: + TOOL_IMAGE: ${{ inputs.tool-image }} + TARGETS: ${{ inputs.targets }} + RUN_CREATED_AT: ${{ inputs.created-at }} run: | - GIT_SHORT_HASH=$(git rev-parse --short HEAD) - if [ "${{ github.ref_type }}" == "tag" ]; then - APP_VERSION=$(echo "${{ github.ref_name }}" | sed 's/^v//') - else - APP_VERSION=$(date '+%Y.%-m.%-d-canary.%-H%M') - fi - if [[ "$APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - BUILD_TYPE=stable - elif [[ "$APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]]; then - BUILD_TYPE=beta - elif [[ "$APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-canary\.[0-9a-f]+$ ]]; then - BUILD_TYPE=canary - else - echo "Error: unsupported version string: $APP_VERSION" >&2 - exit 1 - fi - echo $APP_VERSION - echo $GIT_SHORT_HASH - echo $BUILD_TYPE - echo "APP_VERSION=$APP_VERSION" >> "$GITHUB_OUTPUT" - echo "GIT_SHORT_HASH=$GIT_SHORT_HASH" >> "$GITHUB_OUTPUT" - echo "BUILD_TYPE=$BUILD_TYPE" >> "$GITHUB_OUTPUT" + set -euo pipefail + previous=() + if [ -f release-previous.json ]; then previous=(--previous /work/release-previous.json); fi + docker run --rm --user "$(id -u):$(id -g)" -v "$GITHUB_WORKSPACE:/work" "$TOOL_IMAGE" plan \ + --ref "$GITHUB_REF" --sha "$GITHUB_SHA" --targets "$TARGETS" \ + --request-id "$GITHUB_REPOSITORY_ID-$GITHUB_RUN_ID" --created-at "$RUN_CREATED_AT" --out /work/release-plan.json "${previous[@]}" + { + echo "APP_VERSION=$(jq -r .version release-plan.json)" + echo "GIT_SHORT_HASH=${GITHUB_SHA:0:9}" + echo "BUILD_TYPE=$(jq -r .channel release-plan.json)" + echo "SKIP=$(jq -r ' .skip // false' release-plan.json)" + echo "SERVER=$(jq 'any(.targets[]; .name == "server")' release-plan.json)" + echo "DESKTOP_MATRIX=$(jq -c '{include: [.buildMatrix.include[] | select(.target | startswith("desktop-"))]}' release-plan.json)" + echo "MOBILE_MATRIX=$(jq -c '{include: [.buildMatrix.include[] | select(.target | startswith("mobile-"))]}' release-plan.json)" + } >> "$GITHUB_OUTPUT" diff --git a/.github/deployment/cloud-sql-proxy/Dockerfile b/.github/deployment/cloud-sql-proxy/Dockerfile deleted file mode 100644 index a0237dabb5..0000000000 --- a/.github/deployment/cloud-sql-proxy/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM golang:1.27-bookworm AS build - -ENV CGO_ENABLED=0 -RUN go install -trimpath -ldflags="-s -w" github.com/GoogleCloudPlatform/cloud-sql-proxy/v2@v2.26.2 - -FROM gcr.io/distroless/static-debian12:nonroot -COPY --from=build /go/bin/cloud-sql-proxy /cloud-sql-proxy -ENTRYPOINT ["/cloud-sql-proxy"] diff --git a/.github/helm/affine/.helmignore b/.github/helm/affine/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/.github/helm/affine/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/.github/helm/affine/Chart.yaml b/.github/helm/affine/Chart.yaml deleted file mode 100644 index b33a967adf..0000000000 --- a/.github/helm/affine/Chart.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v2 -name: affine -description: AFFiNE cloud chart -type: application -version: 0.0.0 -appVersion: "0.27.5" -dependencies: - - name: front - version: 0.0.0 - repository: "file://charts/front" - - name: graphql - version: 0.0.0 - repository: "file://charts/graphql" - - name: cloud-sql-proxy - version: 0.0.0 - repository: "file://charts/gcloud-sql-proxy" - condition: global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/front/Chart.yaml b/.github/helm/affine/charts/front/Chart.yaml deleted file mode 100644 index 37df2d0b16..0000000000 --- a/.github/helm/affine/charts/front/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: front -description: AFFiNE front server -type: application -version: 0.0.0 -appVersion: "0.27.5" diff --git a/.github/helm/affine/charts/front/templates/NOTES.txt b/.github/helm/affine/charts/front/templates/NOTES.txt deleted file mode 100644 index 796f60e961..0000000000 --- a/.github/helm/affine/charts/front/templates/NOTES.txt +++ /dev/null @@ -1,16 +0,0 @@ -1. Get the application URL by running these commands: -{{- if contains "NodePort" .Values.services.sync.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ .Values.services.sync.name }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.services.sync.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ .Values.services.sync.name }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ .Values.services.sync.name }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.services.sync.port }} -{{- else if contains "ClusterIP" .Values.services.sync.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "front.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/.github/helm/affine/charts/front/templates/_helpers.tpl b/.github/helm/affine/charts/front/templates/_helpers.tpl deleted file mode 100644 index dcfa4a6cca..0000000000 --- a/.github/helm/affine/charts/front/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "front.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "front.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "front.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "front.labels" -}} -helm.sh/chart: {{ include "front.chart" . }} -{{ include "front.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -monitoring: enabled -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "front.selectorLabels" -}} -app.kubernetes.io/name: {{ include "front.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "front.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "front.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/front/templates/deployment.yaml b/.github/helm/affine/charts/front/templates/deployment.yaml deleted file mode 100644 index c3397bb8dd..0000000000 --- a/.github/helm/affine/charts/front/templates/deployment.yaml +++ /dev/null @@ -1,124 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "front.fullname" . }} - labels: - {{- include "front.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "front.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - checksum/server-config: {{ include "affine.serverConfig" (dict "root" . "port" .Values.app.port) | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "front.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "front.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.global.database.gcloud }} - {{- if .enabled }} - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - {{- end }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - {{- if .Values.global.secret.licenseSecretName }} - - name: AFFINE_PRO_LICENSE_PRIVATE_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.secret.licenseSecretName }}" - key: "{{ .Values.global.secret.licensePrivateKeyKey }}" - {{- end }} - - name: NODE_ENV - value: "{{ .Values.env }}" - - name: NODE_OPTIONS - value: "{{ .Values.nodeOptions }}" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "front" - - name: AFFINE_ENV - value: "{{ .Release.Namespace }}" - ports: - - name: http - containerPort: {{ .Values.app.port }} - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ default .Values.probe.initialDelaySeconds .Values.probe.liveness.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.probe.timeoutSeconds .Values.probe.liveness.timeoutSeconds }} - periodSeconds: {{ default .Values.probe.periodSeconds .Values.probe.liveness.periodSeconds }} - failureThreshold: {{ default .Values.probe.failureThreshold .Values.probe.liveness.failureThreshold }} - successThreshold: {{ default .Values.probe.successThreshold .Values.probe.liveness.successThreshold }} - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ default .Values.probe.initialDelaySeconds .Values.probe.readiness.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.probe.timeoutSeconds .Values.probe.readiness.timeoutSeconds }} - periodSeconds: {{ default .Values.probe.periodSeconds .Values.probe.readiness.periodSeconds }} - failureThreshold: {{ default .Values.probe.failureThreshold .Values.probe.readiness.failureThreshold }} - successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: tmp - mountPath: /tmp - - name: server-config - mountPath: /app/config.json - subPath: front.json - readOnly: true - volumes: - - name: tmp - emptyDir: {} - - name: server-config - secret: - secretName: server-config - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/.github/helm/affine/charts/front/templates/service-renderer.yaml b/.github/helm/affine/charts/front/templates/service-renderer.yaml deleted file mode 100644 index 070f2750f8..0000000000 --- a/.github/helm/affine/charts/front/templates/service-renderer.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.services.renderer.name }} - labels: - {{- include "front.labels" . | nindent 4 }} - {{- with .Values.services.renderer.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.services.renderer.type }} - ports: - - port: {{ .Values.services.renderer.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "front.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/front/templates/service-sync.yaml b/.github/helm/affine/charts/front/templates/service-sync.yaml deleted file mode 100644 index 6db650a498..0000000000 --- a/.github/helm/affine/charts/front/templates/service-sync.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.services.sync.name }} - labels: - {{- include "front.labels" . | nindent 4 }} - {{- $annotations := omit .Values.services.sync.annotations "cloud.google.com/backend-config" }} - annotations: - cloud.google.com/backend-config: {{ dict "default" .Values.services.sync.backendConfig.name | toJson | quote }} - {{- with $annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.services.sync.type }} - ports: - - port: {{ .Values.services.sync.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "front.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/front/templates/service-web.yaml b/.github/helm/affine/charts/front/templates/service-web.yaml deleted file mode 100644 index c98035d218..0000000000 --- a/.github/helm/affine/charts/front/templates/service-web.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.services.web.name }} - labels: - {{- include "front.labels" . | nindent 4 }} - {{- with .Values.services.web.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.services.web.type }} - ports: - - port: {{ .Values.services.web.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "front.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/front/templates/serviceaccount.yaml b/.github/helm/affine/charts/front/templates/serviceaccount.yaml deleted file mode 100644 index 322b290ec2..0000000000 --- a/.github/helm/affine/charts/front/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "front.serviceAccountName" . }} - labels: - {{- include "front.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/front/templates/tests/test-connection.yaml b/.github/helm/affine/charts/front/templates/tests/test-connection.yaml deleted file mode 100644 index d917f44288..0000000000 --- a/.github/helm/affine/charts/front/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "front.fullname" . }}-test-connection" - labels: - {{- include "front.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ .Values.services.sync.name }}:{{ .Values.services.sync.port }}'] - restartPolicy: Never diff --git a/.github/helm/affine/charts/front/values.yaml b/.github/helm/affine/charts/front/values.yaml deleted file mode 100644 index c349c4b5e9..0000000000 --- a/.github/helm/affine/charts/front/values.yaml +++ /dev/null @@ -1,78 +0,0 @@ -replicaCount: 1 -image: - repository: ghcr.io/toeverything/affine - pullPolicy: IfNotPresent - tag: '' - -imagePullSecrets: [] -nameOverride: '' -fullnameOverride: '' -# map to NODE_ENV environment variable -env: 'production' -nodeOptions: '--max-old-space-size=3072' -app: - port: 3010 -serviceAccount: - create: true - annotations: {} - name: 'affine-front' - -podAnnotations: {} - -podSecurityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - -resources: - limits: - cpu: '1' - memory: 6Gi - requests: - cpu: '1' - memory: 4Gi - -probe: - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - liveness: - initialDelaySeconds: 60 - failureThreshold: 12 - readiness: {} - -services: - sync: - name: affine-sync - type: ClusterIP - port: 3010 - backendConfig: - name: affine-sync-backendconfig - timeoutSec: 10800 - annotations: {} - renderer: - name: affine-renderer - type: ClusterIP - port: 3000 - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' - web: - name: affine-web - type: ClusterIP - port: 8080 - annotations: {} -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore b/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml deleted file mode 100644 index 3585ee8108..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: cloud-sql-proxy -description: Google Cloud SQL Proxy -type: application -version: 0.0.0 -appVersion: "2.26.2" diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt b/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt deleted file mode 100644 index b7cb03936a..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.global.database.gcloud.enabled -}} -1. Get the application URL by running these commands: -{{- if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "gcloud-sql-proxy.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "gcloud-sql-proxy.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "gcloud-sql-proxy.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "gcloud-sql-proxy.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl b/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl deleted file mode 100644 index 8d447acbfd..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "gcloud-sql-proxy.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "gcloud-sql-proxy.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "gcloud-sql-proxy.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "gcloud-sql-proxy.labels" -}} -helm.sh/chart: {{ include "gcloud-sql-proxy.chart" . }} -{{ include "gcloud-sql-proxy.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "gcloud-sql-proxy.selectorLabels" -}} -app.kubernetes.io/name: {{ include "gcloud-sql-proxy.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "gcloud-sql-proxy.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "gcloud-sql-proxy.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml deleted file mode 100644 index b66a354a30..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml +++ /dev/null @@ -1,132 +0,0 @@ -{{- if .Values.global.database.gcloud.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "gcloud-sql-proxy.fullname" . }} - labels: - {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "gcloud-sql-proxy.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "gcloud-sql-proxy.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gcloud-sql-proxy.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - "--address" - - "0.0.0.0" - - "--structured-logs" - - "--auto-iam-authn" - - "{{ .Values.database.connectionName }}" - env: - # Enable HTTP healthchecks on port 9801. This enables /liveness, - # /readiness and /startup health check endpoints. Allow connections - # listen for connections on any interface (0.0.0.0) so that the - # k8s management components can reach these endpoints. - - name: CSQL_PROXY_HEALTH_CHECK - value: "true" - - name: CSQL_PROXY_HTTP_PORT - value: "9801" - - name: CSQL_PROXY_HTTP_ADDRESS - value: 0.0.0.0 - ports: - - name: cloud-sql-proxy - containerPort: {{ .Values.service.port }} - protocol: TCP - - containerPort: 9801 - protocol: TCP - # The /startup probe returns OK when the proxy is ready to receive - # connections from the application. In this example, k8s will check - # once a second for 60 seconds. - startupProbe: - failureThreshold: 60 - httpGet: - path: /startup - port: 9801 - scheme: HTTP - periodSeconds: 1 - successThreshold: 1 - timeoutSeconds: 10 - # The /liveness probe returns OK as soon as the proxy application has - # begun its startup process and continues to return OK until the - # process stops. - livenessProbe: - failureThreshold: 3 - httpGet: - path: /liveness - port: 9801 - scheme: HTTP - # The probe will be checked every 10 seconds. - periodSeconds: 10 - # Number of times the probe is allowed to fail before the transition - # from healthy to failure state. - # - # If periodSeconds = 60, 5 tries will result in five minutes of - # checks. The proxy starts to refresh a certificate five minutes - # before its expiration. If those five minutes lapse without a - # successful refresh, the liveness probe will fail and the pod will be - # restarted. - successThreshold: 1 - # The probe will fail if it does not respond in 10 seconds - timeoutSeconds: 10 - readinessProbe: - # The /readiness probe returns OK when the proxy can establish - # a new connections to its databases. - httpGet: - path: /readiness - port: 9801 - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 10 - # Number of times the probe must report success to transition from failure to healthy state. - # Defaults to 1 for readiness probe. - successThreshold: 1 - failureThreshold: 6 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml deleted file mode 100644 index e4f864ea73..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.global.database.gcloud.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "gcloud-sql-proxy.fullname" . }} - labels: - {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.global.database.port }} - targetPort: cloud-sql-proxy - protocol: TCP - name: cloud-sql-proxy - - port: 9801 - targetPort: 9801 - protocol: TCP - name: health - selector: - {{- include "gcloud-sql-proxy.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml deleted file mode 100644 index 5e9bfa2ada..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.global.database.gcloud.enabled -}} -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "gcloud-sql-proxy.serviceAccountName" . }} - labels: - {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} -{{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml deleted file mode 100644 index c6138cc299..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.global.database.gcloud.enabled -}} -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "gcloud-sql-proxy.fullname" . }}-test-connection" - labels: - {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "gcloud-sql-proxy.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never -{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml deleted file mode 100644 index 58e3022cff..0000000000 --- a/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml +++ /dev/null @@ -1,56 +0,0 @@ -replicaCount: 2 -global: - database: - gcloud: - enabled: false -database: - connectionName: "" - -image: - repository: ghcr.io/toeverything/affine - tag: cloud-sql-proxy-2.26.2 - pullPolicy: IfNotPresent - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - create: true - automount: true - annotations: {} - name: "" - -podAnnotations: {} -podLabels: {} - -podSecurityContext: - fsGroup: 2000 - seccompProfile: - type: RuntimeDefault - -securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - -service: - type: ClusterIP - port: 5432 - -resources: - limits: - memory: "1Gi" - cpu: "1" - requests: - memory: "512Mi" - cpu: "100m" - -volumes: [] -volumeMounts: [] -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/.github/helm/affine/charts/graphql/Chart.yaml b/.github/helm/affine/charts/graphql/Chart.yaml deleted file mode 100644 index 9d156ffa8f..0000000000 --- a/.github/helm/affine/charts/graphql/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: graphql -description: AFFiNE GraphQL server -type: application -version: 0.0.0 -appVersion: "0.27.5" diff --git a/.github/helm/affine/charts/graphql/templates/NOTES.txt b/.github/helm/affine/charts/graphql/templates/NOTES.txt deleted file mode 100644 index 2cec931c10..0000000000 --- a/.github/helm/affine/charts/graphql/templates/NOTES.txt +++ /dev/null @@ -1,16 +0,0 @@ -1. Get the application URL by running these commands: -{{- if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "graphql.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "graphql.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "graphql.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "graphql.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/_helpers.tpl b/.github/helm/affine/charts/graphql/templates/_helpers.tpl deleted file mode 100644 index ee3254da8e..0000000000 --- a/.github/helm/affine/charts/graphql/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "graphql.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "graphql.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "graphql.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "graphql.labels" -}} -helm.sh/chart: {{ include "graphql.chart" . }} -{{ include "graphql.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -monitoring: enabled -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "graphql.selectorLabels" -}} -app.kubernetes.io/name: {{ include "graphql.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "graphql.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "graphql.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml deleted file mode 100644 index 8a419c362c..0000000000 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ /dev/null @@ -1,124 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "graphql.fullname" . }} - labels: - {{- include "graphql.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "graphql.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - checksum/server-config: {{ include "affine.serverConfig" (dict "root" . "port" .Values.service.port) | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "graphql.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "graphql.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.global.database.gcloud }} - {{- if .enabled }} - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - {{- end }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - {{- if .Values.global.secret.licenseSecretName }} - - name: AFFINE_PRO_LICENSE_PRIVATE_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.secret.licenseSecretName }}" - key: "{{ .Values.global.secret.licensePrivateKeyKey }}" - {{- end }} - - name: NODE_ENV - value: "{{ .Values.env }}" - - name: NODE_OPTIONS - value: "--max-old-space-size=2048" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "graphql" - - name: AFFINE_ENV - value: "{{ .Release.Namespace }}" - ports: - - name: http - containerPort: {{ .Values.service.port }} - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.probe.timeoutSeconds .Values.probe.liveness.timeoutSeconds }} - periodSeconds: {{ default .Values.probe.periodSeconds .Values.probe.liveness.periodSeconds }} - failureThreshold: {{ default .Values.probe.failureThreshold .Values.probe.liveness.failureThreshold }} - successThreshold: {{ default .Values.probe.successThreshold .Values.probe.liveness.successThreshold }} - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.probe.timeoutSeconds .Values.probe.readiness.timeoutSeconds }} - periodSeconds: {{ default .Values.probe.periodSeconds .Values.probe.readiness.periodSeconds }} - failureThreshold: {{ default .Values.probe.failureThreshold .Values.probe.readiness.failureThreshold }} - successThreshold: {{ default .Values.probe.successThreshold .Values.probe.readiness.successThreshold }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: tmp - mountPath: /tmp - - name: server-config - mountPath: /app/config.json - subPath: graphql.json - readOnly: true - volumes: - - name: tmp - emptyDir: {} - - name: server-config - secret: - secretName: server-config - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml deleted file mode 100644 index 31c3f4d995..0000000000 --- a/.github/helm/affine/charts/graphql/templates/migration.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "graphql.fullname" . }}-database-migration - labels: - {{- include "graphql.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": post-install,pre-upgrade - "helm.sh/hook-weight": "-1" - "helm.sh/hook-delete-policy": before-hook-creation - -spec: - template: - spec: - serviceAccountName: {{ include "graphql.serviceAccountName" . }} - {{- with .Values.global.database.gcloud }} - {{- if .enabled }} - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - {{- end }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - command: ["yarn", "predeploy"] - env: - # Prisma CLI still reads DATABASE_URL; derive it from the same config. - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "{{ .Values.env }}" - - name: AFFINE_ENV - value: "{{ .Release.Namespace }}" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - volumeMounts: - - name: server-config - mountPath: /app/config.json - subPath: graphql.json - readOnly: true - resources: - requests: - cpu: '100m' - memory: '200Mi' - volumes: - - name: server-config - secret: - secretName: server-config - restartPolicy: Never - backoffLimit: 1 diff --git a/.github/helm/affine/charts/graphql/templates/service.yaml b/.github/helm/affine/charts/graphql/templates/service.yaml deleted file mode 100644 index 4c34622c41..0000000000 --- a/.github/helm/affine/charts/graphql/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "graphql.fullname" . }} - labels: - {{- include "graphql.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "graphql.selectorLabels" . | nindent 4 }} diff --git a/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml b/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml deleted file mode 100644 index 14dac586bf..0000000000 --- a/.github/helm/affine/charts/graphql/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "graphql.serviceAccountName" . }} - labels: - {{- include "graphql.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml b/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml deleted file mode 100644 index 6e46e00d71..0000000000 --- a/.github/helm/affine/charts/graphql/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "graphql.fullname" . }}-test-connection" - labels: - {{- include "graphql.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "graphql.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml deleted file mode 100644 index 58f22727db..0000000000 --- a/.github/helm/affine/charts/graphql/values.yaml +++ /dev/null @@ -1,54 +0,0 @@ -replicaCount: 1 -image: - repository: ghcr.io/toeverything/affine - pullPolicy: IfNotPresent - tag: '' - -imagePullSecrets: [] -nameOverride: '' -fullnameOverride: '' -# map to NODE_ENV environment variable -env: 'production' -serviceAccount: - create: true - annotations: {} - name: 'affine-graphql' - -podAnnotations: {} - -podSecurityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - -securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - -resources: - limits: - cpu: '1' - memory: 4Gi - requests: - cpu: '1' - memory: 2Gi - -probe: - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - liveness: - failureThreshold: 12 - readiness: {} - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/.github/helm/affine/templates/_helpers.tpl b/.github/helm/affine/templates/_helpers.tpl deleted file mode 100644 index 2182186c5a..0000000000 --- a/.github/helm/affine/templates/_helpers.tpl +++ /dev/null @@ -1,77 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "affine.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "affine.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "affine.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "affine.labels" -}} -helm.sh/chart: {{ include "affine.chart" . }} -{{ include "affine.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "affine.selectorLabels" -}} -app.kubernetes.io/name: {{ include "affine.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "affine.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "affine.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Render the server runtime configuration shared by the Secret and Pod checksums. -*/}} -{{- define "affine.serverConfig" -}} -{{- $config := deepCopy (required "global.serverConfig is required" .root.Values.global.serverConfig) -}} -{{- if ne (dig "deployment" "type" "" $config) "cloud" -}} -{{- fail "This chart requires deployment.type=cloud" -}} -{{- end -}} -{{- if not (hasKey $config "server") -}} -{{- $_ := set $config "server" dict -}} -{{- end -}} -{{- $_ := set $config.server "port" .port -}} -{{- $config | toJson -}} -{{- end }} diff --git a/.github/helm/affine/templates/backend-config.yaml b/.github/helm/affine/templates/backend-config.yaml deleted file mode 100644 index 775b00a555..0000000000 --- a/.github/helm/affine/templates/backend-config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: cloud.google.com/v1 -kind: BackendConfig -metadata: - name: affine-api-backendconfig - annotations: - meta.helm.sh/release-name: {{ .Release.Name }} - meta.helm.sh/release-namespace: {{ .Release.Namespace }} - labels: - {{- include "affine.labels" . | nindent 4 }} -spec: - healthCheck: - timeoutSec: 1 - type: HTTP - requestPath: /info ---- -apiVersion: cloud.google.com/v1 -kind: BackendConfig -metadata: - name: {{ .Values.front.services.sync.backendConfig.name }} - annotations: - meta.helm.sh/release-name: {{ .Release.Name }} - meta.helm.sh/release-namespace: {{ .Release.Namespace }} - labels: - {{- include "affine.labels" . | nindent 4 }} -spec: - timeoutSec: {{ .Values.front.services.sync.backendConfig.timeoutSec }} - healthCheck: - timeoutSec: 1 - type: HTTP - requestPath: /info diff --git a/.github/helm/affine/templates/configmap.yaml b/.github/helm/affine/templates/configmap.yaml deleted file mode 100644 index ce0603f86d..0000000000 --- a/.github/helm/affine/templates/configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-runtime-config -data: - web-assets-manifest: |- - {{ .Files.Get "web-assets-manifest.json" | nindent 4 }} - mobile-assets-manifest: |- - {{ .Files.Get "mobile-assets-manifest.json" | nindent 4 }} diff --git a/.github/helm/affine/templates/ingress.yaml b/.github/helm/affine/templates/ingress.yaml deleted file mode 100644 index 1c5bbcf414..0000000000 --- a/.github/helm/affine/templates/ingress.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{{- if .Values.global.ingress.enabled -}} -{{- $fullName := include "affine.fullname" . -}} -{{- if and .Values.global.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.global.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.global.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "affine.labels" . | nindent 4 }} - {{- with .Values.global.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.global.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.global.ingress.className }} - {{- end }} - {{- if .Values.global.ingress.tls }} - tls: - {{- range .Values.global.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.global.ingress.hosts }} - - host: {{ . | quote }} - http: - paths: - - path: /socket.io - pathType: Prefix - backend: - service: - name: {{ $.Values.front.services.sync.name }} - port: - number: {{ $.Values.front.services.sync.port }} - - path: /graphql - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: {{ $.Values.graphql.service.port }} - - path: /api - pathType: Prefix - backend: - service: - name: affine-graphql - port: - number: {{ $.Values.graphql.service.port }} - - path: /workspace - pathType: Prefix - backend: - service: - name: {{ $.Values.front.services.renderer.name }} - port: - number: {{ $.Values.front.services.renderer.port }} - - path: / - pathType: Prefix - backend: - service: - name: {{ $.Values.front.services.web.name }} - port: - number: {{ $.Values.front.services.web.port }} - {{- end }} -{{- end }} diff --git a/.github/helm/affine/templates/monitoring.yaml b/.github/helm/affine/templates/monitoring.yaml deleted file mode 100644 index 8a84faa243..0000000000 --- a/.github/helm/affine/templates/monitoring.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: monitoring.googleapis.com/v1 -kind: PodMonitoring -metadata: - name: "{{ .Release.Name }}-monitoring" -spec: - selector: - matchLabels: - app.kubernetes.io/instance: {{ .Release.Name }} - endpoints: - - port: 9464 - interval: 30s diff --git a/.github/helm/affine/templates/server-config-secret.yaml b/.github/helm/affine/templates/server-config-secret.yaml deleted file mode 100644 index 9980ccd771..0000000000 --- a/.github/helm/affine/templates/server-config-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: server-config - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-2" - "helm.sh/hook-delete-policy": before-hook-creation -type: Opaque -data: - front.json: {{ include "affine.serverConfig" (dict "root" . "port" .Values.front.app.port) | b64enc | quote }} - graphql.json: {{ include "affine.serverConfig" (dict "root" . "port" .Values.graphql.service.port) | b64enc | quote }} - worker.json: {{ include "affine.serverConfig" (dict "root" . "port" .Values.worker.app.port) | b64enc | quote }} - database-url: {{ required "serverConfig.db.datasourceUrl is required" .Values.global.serverConfig.db.datasourceUrl | b64enc | quote }} diff --git a/.github/helm/affine/templates/worker-deployment.yaml b/.github/helm/affine/templates/worker-deployment.yaml deleted file mode 100644 index 5489735bfc..0000000000 --- a/.github/helm/affine/templates/worker-deployment.yaml +++ /dev/null @@ -1,119 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: affine-worker - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - monitoring: enabled -spec: - replicas: {{ .Values.worker.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - annotations: - checksum/server-config: {{ include "affine.serverConfig" (dict "root" . "port" .Values.worker.app.port) | sha256sum }} - {{- with .Values.worker.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - serviceAccountName: {{ .Values.worker.serviceAccount.name }} - securityContext: - {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} - {{- with .Values.global.database.gcloud }} - {{- if .enabled }} - initContainers: - - name: wait-for-cloud-sql-proxy - image: busybox:1.36.1 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ec - - | - until wget -q -T 2 -O /dev/null "http://{{ $.Values.global.database.host }}:9801/startup"; do - echo "waiting for cloud sql proxy to become ready" - sleep 2 - done - {{- end }} - {{- end }} - containers: - - name: worker - image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Values.graphql.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: IfNotPresent - securityContext: - {{- toYaml .Values.worker.securityContext | nindent 12 }} - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: server-config - key: database-url - - name: NODE_ENV - value: "{{ .Values.worker.env }}" - - name: NODE_OPTIONS - value: "{{ .Values.worker.nodeOptions }}" - - name: NO_COLOR - value: "1" - - name: DEPLOYMENT_PLATFORM - value: "gcp" - - name: SERVER_FLAVOR - value: "worker" - - name: AFFINE_ENV - value: "{{ .Release.Namespace }}" - ports: - - name: http - containerPort: {{ .Values.worker.app.port }} - protocol: TCP - livenessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.liveness.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.liveness.timeoutSeconds }} - periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.liveness.periodSeconds }} - failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.liveness.failureThreshold }} - successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.liveness.successThreshold }} - readinessProbe: - httpGet: - path: /info - port: http - initialDelaySeconds: {{ default .Values.worker.probe.initialDelaySeconds .Values.worker.probe.readiness.initialDelaySeconds }} - timeoutSeconds: {{ default .Values.worker.probe.timeoutSeconds .Values.worker.probe.readiness.timeoutSeconds }} - periodSeconds: {{ default .Values.worker.probe.periodSeconds .Values.worker.probe.readiness.periodSeconds }} - failureThreshold: {{ default .Values.worker.probe.failureThreshold .Values.worker.probe.readiness.failureThreshold }} - successThreshold: {{ default .Values.worker.probe.successThreshold .Values.worker.probe.readiness.successThreshold }} - resources: - {{- toYaml .Values.worker.resources | nindent 12 }} - volumeMounts: - - name: server-config - mountPath: /app/config.json - subPath: worker.json - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: server-config - secret: - secretName: server-config - - name: tmp - emptyDir: {} - {{- with .Values.worker.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.worker.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.worker.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/.github/helm/affine/templates/worker-serviceaccount.yaml b/.github/helm/affine/templates/worker-serviceaccount.yaml deleted file mode 100644 index 05c8447a6d..0000000000 --- a/.github/helm/affine/templates/worker-serviceaccount.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.worker.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.worker.serviceAccount.name }} - labels: - app.kubernetes.io/name: affine-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - {{- with .Values.worker.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml deleted file mode 100644 index 81bbd30a58..0000000000 --- a/.github/helm/affine/values.yaml +++ /dev/null @@ -1,102 +0,0 @@ -global: - app: - buildType: 'stable' - ingress: - enabled: false - className: '' - # hosts for ingress rules - # e.g. - # hosts: - # - affine.pro - # - www.affine.pro - hosts: - - affine.pro - tls: [] - secret: - # Name of an externally managed Secret containing the license signing key. - # The chart never generates this key. Leave empty to disable license issuance. - licenseSecretName: '' - licensePrivateKeyKey: 'license-private-key' - database: - gcloud: - enabled: false - host: 'affine-cloud-sql-proxy' - port: '5432' - # Complete server configuration, supplied by the deployment environment. - serverConfig: {} - -graphql: - service: - type: ClusterIP - port: 3000 - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' - -front: - services: - sync: - name: affine-sync - type: ClusterIP - port: 3010 - backendConfig: - name: affine-sync-backendconfig - timeoutSec: 10800 - annotations: {} - renderer: - name: affine-renderer - type: ClusterIP - port: 3000 - annotations: - cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}' - web: - name: affine-web - type: ClusterIP - port: 8080 - -worker: - replicaCount: 1 - image: - repository: ghcr.io/toeverything/affine - tag: '' - env: 'production' - nodeOptions: '--max-old-space-size=3072' - app: - port: 3002 - podAnnotations: {} - podSecurityContext: - fsGroup: 2000 - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - serviceAccount: - create: true - annotations: {} - name: 'affine-worker' - resources: - limits: - cpu: '1' - memory: 4Gi - requests: - cpu: '1' - memory: 2Gi - probe: - initialDelaySeconds: 20 - timeoutSeconds: 5 - periodSeconds: 10 - failureThreshold: 6 - successThreshold: 1 - liveness: - initialDelaySeconds: 60 - failureThreshold: 12 - readiness: {} - nodeSelector: {} - tolerations: [] - affinity: {} diff --git a/.github/helm/releaser.yaml b/.github/helm/releaser.yaml deleted file mode 100644 index e95657e987..0000000000 --- a/.github/helm/releaser.yaml +++ /dev/null @@ -1,2 +0,0 @@ -owner: toeverything -git-repo: helm-charts diff --git a/.github/helm/separate-config/Dockerfile_pgvector b/.github/helm/separate-config/Dockerfile_pgvector deleted file mode 100644 index 1999273a6c..0000000000 --- a/.github/helm/separate-config/Dockerfile_pgvector +++ /dev/null @@ -1,6 +0,0 @@ -FROM pgvector/pgvector:pg15 AS builder - -FROM bitnami/postgresql:15 - -COPY --from=builder /usr/lib/postgresql/15/lib/vector.so /opt/bitnami/postgresql/lib/ -COPY --from=builder /usr/share/postgresql/15/extension/vector* /opt/bitnami/postgresql/share/extension/ \ No newline at end of file diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 545354b98f..0bee1fca81 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -1,6 +1,5 @@ -name: Build Images - -on: +name: Build Server Candidates +'on': workflow_call: inputs: build-type: @@ -12,17 +11,17 @@ on: git-short-hash: type: string required: true - + outputs: + image: + description: Immutable candidate image + value: ${{ jobs.build-images.outputs.image }} permissions: - contents: 'write' - id-token: 'write' - packages: 'write' - + contents: read + packages: write jobs: build-web: name: Build @affine/web runs-on: ubuntu-latest - environment: ${{ inputs.build-type }} steps: - uses: actions/checkout@v7 - name: Setup Version @@ -34,12 +33,10 @@ jobs: - name: Build Core run: yarn affine @affine/web build env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} BUILD_TYPE: ${{ inputs.build-type }} + RELEASE_ASSET_PREFIX: releases/${{ inputs.app-version }}/ SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-web' + SENTRY_PROJECT: affine-web SENTRY_RELEASE: ${{ inputs.app-version }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} @@ -47,14 +44,13 @@ jobs: - name: Upload web artifact uses: actions/upload-artifact@v4 with: + overwrite: true name: web path: ./packages/frontend/apps/web/dist if-no-files-found: error - build-admin: name: Build @affine/admin runs-on: ubuntu-latest - environment: ${{ inputs.build-type }} steps: - uses: actions/checkout@v7 - name: Setup Version @@ -66,26 +62,23 @@ jobs: - name: Build Admin run: yarn affine @affine/admin build env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} BUILD_TYPE: ${{ inputs.build-type }} + RELEASE_ASSET_PREFIX: releases/${{ inputs.app-version }}/ SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-admin' + SENTRY_PROJECT: affine-admin SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - name: Upload admin artifact uses: actions/upload-artifact@v4 with: + overwrite: true name: admin path: ./packages/frontend/admin/dist if-no-files-found: error - build-mobile: name: Build @affine/mobile runs-on: ubuntu-latest - environment: ${{ inputs.build-type }} steps: - uses: actions/checkout@v7 - name: Setup Version @@ -97,26 +90,23 @@ jobs: - name: Build Mobile run: yarn affine @affine/mobile build env: - R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} - R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} BUILD_TYPE: ${{ inputs.build-type }} + RELEASE_ASSET_PREFIX: releases/${{ inputs.app-version }}/ SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine-mobile' + SENTRY_PROJECT: affine-mobile SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} PERFSEE_TOKEN: ${{ secrets.PERFSEE_TOKEN }} - name: Upload mobile artifact uses: actions/upload-artifact@v4 with: + overwrite: true name: mobile path: ./packages/frontend/apps/mobile/dist if-no-files-found: error - build-server-native: name: Build Server native - ${{ matrix.targets.name }} runs-on: ubuntu-22.04 - environment: ${{ inputs.build-type }} strategy: fail-fast: false matrix: @@ -127,7 +117,6 @@ jobs: file: server-native.arm64.node - name: armv7-unknown-linux-gnueabihf file: server-native.armv7.node - steps: - uses: actions/checkout@v7 - name: Setup Version @@ -148,15 +137,14 @@ jobs: target: ${{ matrix.targets.name }} package: '@affine/server-native' - name: Rename ${{ matrix.targets.file }} - run: | - mv ./packages/backend/native/server-native.node ./packages/backend/native/${{ matrix.targets.file }} + run: mv ./packages/backend/native/server-native.node ./packages/backend/native/${{ matrix.targets.file }} - name: Upload ${{ matrix.targets.file }} uses: actions/upload-artifact@v4 with: + overwrite: true name: server-native-${{ matrix.targets.file }} path: ./packages/backend/native/${{ matrix.targets.file }} if-no-files-found: error - build-server: name: Build Server runs-on: ubuntu-latest @@ -186,10 +174,10 @@ jobs: - name: Upload server dist uses: actions/upload-artifact@v4 with: + overwrite: true name: server-dist path: ./packages/backend/server/dist if-no-files-found: error - build-images: name: Build Images runs-on: ubuntu-latest @@ -216,61 +204,40 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # setup node without cache configuration - # Prisma cache is not compatible with docker build cache - name: Setup Node.js uses: actions/setup-node@v7 with: - node-version-file: '.nvmrc' + node-version-file: .nvmrc registry-url: https://npm.pkg.github.com scope: '@toeverything' - - name: Download web artifact uses: actions/download-artifact@v4 with: name: web path: ./packages/frontend/apps/web/dist - - name: Download mobile artifact uses: actions/download-artifact@v4 with: name: mobile path: ./packages/frontend/apps/mobile/dist - - name: Download admin artifact uses: actions/download-artifact@v4 with: name: admin path: ./packages/frontend/admin/dist - - name: Install Node.js dependencies run: | yarn config set --json supportedArchitectures.cpu '["x64", "arm64", "arm"]' yarn config set --json supportedArchitectures.libc '["glibc"]' yarn workspaces focus @affine/server --production - - name: Generate Prisma client run: yarn workspace @affine/server prisma generate - - name: Mv node_modules run: mv ./node_modules ./packages/backend/server - - name: Setup Version uses: ./.github/actions/setup-version with: app-version: ${{ inputs.app-version }} - - - name: Build Cloud SQL Proxy with patched Go - uses: docker/build-push-action@v6 - with: - context: . - push: true - pull: true - platforms: linux/amd64,linux/arm64 - provenance: true - file: .github/deployment/cloud-sql-proxy/Dockerfile - tags: ghcr.io/toeverything/affine:cloud-sql-proxy-2.26.2 - - name: Build backend Dockerfile uses: docker/build-push-action@v6 with: @@ -280,4 +247,7 @@ jobs: platforms: linux/amd64,linux/arm64,linux/arm/v7 provenance: true file: .github/deployment/node/Dockerfile - tags: ghcr.io/toeverything/affine:${{inputs.build-type}}-${{ inputs.git-short-hash }} + tags: ghcr.io/toeverything/affine:candidate-${{ github.run_id }}-${{ github.run_attempt }} + id: image + outputs: + image: ghcr.io/toeverything/affine@${{ steps.image.outputs.digest }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 96dddf27f5..5fd7c9d6d8 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -488,7 +488,7 @@ jobs: e2e-blocksuite-test: name: E2E BlockSuite Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -520,7 +520,7 @@ jobs: e2e-blocksuite-cross-browser-test: name: E2E BlockSuite Cross Browser Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - name: Setup Node.js @@ -586,7 +586,7 @@ jobs: e2e-mobile-test: name: E2E Mobile Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: DISTRIBUTION: mobile IN_CI_TEST: true @@ -617,7 +617,7 @@ jobs: unit-test: name: Unit Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - build-native-linux env: @@ -656,7 +656,7 @@ jobs: build-native-linux: name: Build AFFiNE native (x86_64-unknown-linux-gnu) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '16' CARGO_PROFILE_RELEASE_DEBUG: '1' @@ -782,7 +782,7 @@ jobs: build-server-native: name: Build Server native - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: AFFINE_PRO_PUBLIC_KEY: | -----BEGIN PUBLIC KEY----- @@ -1381,7 +1381,7 @@ jobs: copilot-e2e-test: name: Frontend Copilot E2E Test if: ${{ needs.copilot-test-filter.outputs.run-e2e == 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: DISTRIBUTION: web DATABASE_URL: postgresql://affine:affine@localhost:5432/affine @@ -1442,7 +1442,7 @@ jobs: cloud-e2e-test: name: ${{ matrix.tests.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - build-server-native - build-native-linux @@ -1559,7 +1559,7 @@ jobs: test: true, } - { - os: ubuntu-latest, + os: ubuntu-24.04, platform: linux, arch: x64, target: x86_64-unknown-linux-gnu, @@ -1611,7 +1611,7 @@ jobs: run: yarn affine @affine/electron build - name: Run desktop tests - if: ${{ matrix.spec.os == 'ubuntu-latest' }} + if: ${{ matrix.spec.os == 'ubuntu-24.04' }} run: | # Workaround for Electron apps failing to initialize on Ubuntu 24.04 due to AppArmor restrictions # Disables unprivileged user namespaces restriction to allow Electron apps to run @@ -1620,7 +1620,7 @@ jobs: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn affine @affine-test/affine-desktop e2e - name: Run desktop tests - if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-latest' }} + if: ${{ matrix.spec.test && matrix.spec.os != 'ubuntu-24.04' }} run: yarn affine @affine-test/affine-desktop e2e - name: Make bundle (macOS) diff --git a/.github/workflows/copilot-test.yml b/.github/workflows/copilot-test.yml index e15b1f6ead..d857d950b9 100644 --- a/.github/workflows/copilot-test.yml +++ b/.github/workflows/copilot-test.yml @@ -6,7 +6,7 @@ on: jobs: build-server-native: name: Build Server native - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '16' CARGO_PROFILE_RELEASE_DEBUG: '1' @@ -32,7 +32,7 @@ jobs: copilot-api-test: name: Server Copilot Api Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - build-server-native env: @@ -98,7 +98,7 @@ jobs: copilot-e2e-test: name: Frontend Copilot E2E Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: DISTRIBUTION: web DATABASE_URL: postgresql://affine:affine@localhost:5432/affine diff --git a/.github/workflows/release-cloud.yml b/.github/workflows/release-cloud.yml deleted file mode 100644 index d833c0b777..0000000000 --- a/.github/workflows/release-cloud.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Release Cloud - -on: - workflow_call: - inputs: - build-type: - required: true - type: string - app-version: - required: true - type: string - git-short-hash: - required: true - type: string - -permissions: - contents: 'write' - id-token: 'write' - packages: 'write' - -jobs: - build-images: - name: Build Images - uses: ./.github/workflows/build-images.yml - secrets: inherit - with: - build-type: ${{ inputs.build-type }} - app-version: ${{ inputs.app-version }} - git-short-hash: ${{ inputs.git-short-hash }} - - deploy: - name: Deploy to cluster - environment: ${{ inputs.build-type }} - needs: - - build-images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Deploy to ${{ inputs.build-type }} - uses: ./.github/actions/deploy - with: - gcp-project-number: ${{ secrets.GCP_PROJECT_NUMBER }} - gcp-project-id: ${{ secrets.GCP_PROJECT_ID }} - service-account: ${{ secrets.GCP_HELM_DEPLOY_SERVICE_ACCOUNT }} - cluster-name: ${{ secrets.GCP_CLUSTER_NAME }} - cluster-location: ${{ secrets.GCP_CLUSTER_LOCATION }} - env: - BUILD_TYPE: ${{ inputs.build-type }} - APP_VERSION: ${{ inputs.app-version }} - GIT_SHORT_HASH: ${{ inputs.git-short-hash }} - SERVER_CONFIG: ${{ secrets.SERVER_CONFIG }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GCLOUD_CONNECTION_NAME: ${{ secrets.GCLOUD_CONNECTION_NAME }} - CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }} - APP_IAM_ACCOUNT: ${{ secrets.APP_IAM_ACCOUNT }} - STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }} diff --git a/.github/workflows/release-desktop-platform.yml b/.github/workflows/release-desktop-platform.yml index 04ad715f8d..21dd1e21e0 100644 --- a/.github/workflows/release-desktop-platform.yml +++ b/.github/workflows/release-desktop-platform.yml @@ -1,6 +1,5 @@ -name: Release Desktop Platform - -on: +name: Build Desktop Candidate +'on': workflow_call: inputs: build_type: @@ -12,70 +11,33 @@ on: git_short_hash: required: true type: string - runner: - required: true + matrix: type: string - platform: required: true - type: string - arch: - required: true - type: string - target: - required: true - type: string - apple_codesign: - required: false - default: false - type: boolean - install_linux_deps: - required: false - default: false - type: boolean - enable_scripts: - required: false - default: false - type: boolean - outputs: - files_to_be_signed: - description: Files to be signed (Windows only) - value: ${{ jobs.build.outputs.files_to_be_signed }} - permissions: - actions: write - contents: write - security-events: write - id-token: write - attestations: write - + contents: read jobs: build: - runs-on: ${{ inputs.runner }} - outputs: - files_to_be_signed: ${{ steps.get_files_to_be_signed.outputs.FILES_TO_BE_SIGNED }} + runs-on: ${{ matrix.runner }} env: BUILD_TYPE: ${{ inputs.build_type }} RELEASE_VERSION: ${{ inputs.app_version }} - DEBUG: 'affine:*,napi:*' + DEBUG: affine:*,napi:* APP_NAME: affine MACOSX_DEPLOYMENT_TARGET: '12.0' SKIP_GENERATE_ASSETS: 1 - APPLE_ID: ${{ secrets.APPLE_ID }} - APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} - APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine' + SENTRY_PROJECT: affine SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} SENTRY_RELEASE: ${{ inputs.app_version }} + AFFINE_UNSIGNED_PACKAGE: '1' steps: - uses: actions/checkout@v7 - - name: Setup Version uses: ./.github/actions/setup-version with: app-version: ${{ inputs.app_version }} - - name: Setup Node.js timeout-minutes: 10 uses: ./.github/actions/setup-node @@ -83,143 +45,55 @@ jobs: extra-flags: workspaces focus @affine/electron @affine/monorepo @affine/nbstore @toeverything/infra hard-link-nm: false nmHoistingLimits: workspaces - enableScripts: ${{ inputs.enable_scripts }} - + enableScripts: true - name: Build AFFiNE native uses: ./.github/actions/build-rust with: - target: ${{ inputs.target }} + target: ${{ matrix.rustTarget }} package: '@affine/native' - - uses: actions/download-artifact@v4 with: name: desktop-web path: packages/frontend/apps/electron/resources/web-static - - name: Build Desktop Layers run: yarn affine @affine/electron build - - - name: Signing By Apple Developer ID - if: ${{ inputs.platform == 'darwin' && inputs.apple_codesign }} - uses: apple-actions/import-codesign-certs@v7 - with: - p12-file-base64: ${{ secrets.CERTIFICATES_P12 }} - p12-password: ${{ secrets.CERTIFICATES_P12_PASSWORD }} - - - name: Install additional dependencies on Linux - if: ${{ inputs.platform == 'linux' && inputs.install_linux_deps }} - run: | - df -h - sudo add-apt-repository universe - sudo apt install -y libfuse2 elfutils flatpak flatpak-builder - flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo - flatpak update - # some flatpak deps need git protocol.file.allow - git config --global protocol.file.allow always - # clean up apt cache to save disk space - sudo -E apt-get -y purge azure-cli* zulu* hhvm* llvm* firefox* google* dotnet* aspnetcore* powershell* adoptopenjdk* mysql* php* mongodb* moby* snap* || true - sudo -E apt-get -qq autoremove --purge - sudo rm -rf /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/lib/android - sudo apt-get clean - rm -rf ~/.cache/yarn ~/.npm - df -h - - name: Remove nbstore node_modules (darwin/linux) - if: ${{ inputs.platform != 'win32' }} + if: ${{ matrix.platform != 'win32' }} shell: bash - # node_modules of nbstore is not needed for building, and it will make the build process out of memory - run: | - cargo clean + run: 'cargo clean + rm -rf packages/frontend/apps/electron/node_modules/@affine/nbstore/node_modules/@blocksuite + rm -rf packages/frontend/apps/electron/node_modules/@affine/native/node_modules + ' - name: Remove nbstore node_modules (windows) - if: ${{ inputs.platform == 'win32' }} + if: ${{ matrix.platform == 'win32' }} shell: bash - run: | - rm -rf packages/frontend/apps/electron/node_modules/@affine/nbstore/node_modules/@blocksuite/affine/node_modules + run: + 'rm -rf packages/frontend/apps/electron/node_modules/@affine/nbstore/node_modules/@blocksuite/affine/node_modules + rm -rf packages/frontend/apps/electron/node_modules/@affine/native/node_modules - - name: make - if: ${{ inputs.platform != 'win32' }} - run: yarn affine @affine/electron make --platform=${{ inputs.platform }} --arch=${{ inputs.arch }} + ' + - name: Package unsigned application + run: yarn affine @affine/electron package --platform=${{ matrix.platform }} --arch=${{ matrix.arch }} env: - SKIP_WEB_BUILD: 1 - HOIST_NODE_MODULES: 1 + SKIP_WEB_BUILD: '1' + HOIST_NODE_MODULES: '1' NODE_OPTIONS: --max-old-space-size=14384 - - - name: package - if: ${{ inputs.platform == 'win32' }} - run: | - yarn affine @affine/electron package --platform=${{ inputs.platform }} --arch=${{ inputs.arch }} - env: - SKIP_WEB_BUILD: 1 - HOIST_NODE_MODULES: 1 - NODE_OPTIONS: --max-old-space-size=14384 - - - name: signing DMG - if: ${{ inputs.platform == 'darwin' && inputs.apple_codesign }} - run: | - codesign --force --sign "Developer ID Application: TOEVERYTHING PTE. LTD." packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make/AFFiNE.dmg - - - name: Save artifacts (mac) - if: ${{ inputs.platform == 'darwin' }} - run: | - mkdir -p builds - mv packages/frontend/apps/electron/out/*/make/*.dmg ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ inputs.arch }}.dmg - mv packages/frontend/apps/electron/out/*/make/zip/darwin/${{ inputs.arch }}/*.zip ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ inputs.arch }}.zip - - - name: Save artifacts (linux) - if: ${{ inputs.platform == 'linux' }} - run: | - mkdir -p builds - mv packages/frontend/apps/electron/out/*/make/zip/linux/${{ inputs.arch }}/*.zip ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.zip - mv packages/frontend/apps/electron/out/*/make/AppImage/${{ inputs.arch }}/*.AppImage ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.appimage - mv packages/frontend/apps/electron/out/*/make/deb/${{ inputs.arch }}/*.deb ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.deb - mv packages/frontend/apps/electron/out/*/make/flatpak/*/*.flatpak ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.flatpak - - - uses: actions/attest-build-provenance@v4 - if: ${{ inputs.platform == 'darwin' }} + - name: Archive packaged application + shell: bash + run: + tar -czf candidate-${{ matrix.platform }}-${{ matrix.arch }}.tar.gz -C packages/frontend/apps/electron + out resources + - uses: actions/upload-artifact@v4 with: - subject-path: | - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ inputs.arch }}.zip - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-macos-${{ inputs.arch }}.dmg - - - uses: actions/attest-build-provenance@v4 - if: ${{ inputs.platform == 'linux' }} - with: - subject-path: | - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.zip - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.appimage - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.deb - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-linux-${{ inputs.arch }}.flatpak - - - name: Upload Artifact - if: ${{ inputs.platform == 'darwin' || inputs.platform == 'linux' }} - uses: actions/upload-artifact@v4 - with: - name: affine-${{ inputs.platform }}-${{ inputs.arch }}-builds - path: builds - - - name: get all files to be signed - id: get_files_to_be_signed - if: ${{ inputs.platform == 'win32' }} - shell: pwsh - run: | - Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/apps/electron/out -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\apps\electron\out\', '') + '"' }) -join ' ') - "FILES_TO_BE_SIGNED=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT - echo $FILES_TO_BE_SIGNED - - - name: Zip artifacts for faster upload - if: ${{ inputs.platform == 'win32' }} - shell: pwsh - run: Compress-Archive -CompressionLevel Fastest -Path packages/frontend/apps/electron/out/* -DestinationPath archive.zip - - - name: Save packaged artifacts for signing - if: ${{ inputs.platform == 'win32' }} - uses: actions/upload-artifact@v4 - with: - name: packaged-${{ inputs.platform }}-${{ inputs.arch }} - path: | - archive.zip - !**/*.map + overwrite: true + name: candidate-desktop-${{ matrix.platform }}-${{ matrix.arch }} + path: candidate-${{ matrix.platform }}-${{ matrix.arch }}.tar.gz + if-no-files-found: error + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + continue-on-error: ${{ !matrix.required }} diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 4257dc3a08..cc5d16873a 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -1,6 +1,5 @@ -name: Release Desktop - -on: +name: Build Desktop Candidates +'on': workflow_call: inputs: build-type: @@ -12,46 +11,20 @@ on: git-short-hash: required: true type: string - desktop_macos: - description: 'Desktop - macOS' - required: false - default: true - type: boolean - desktop_windows: - description: 'Desktop - Windows' - required: false - default: true - type: boolean - desktop_linux: - description: 'Desktop - Linux' - required: false - default: true - type: boolean - require-windows-signing: - description: 'Require all Windows signing steps to succeed before release' - required: false - default: false - type: boolean - + matrix: + required: true + type: string permissions: - actions: write - contents: write - security-events: write - id-token: write - attestations: write - + contents: read env: BUILD_TYPE: ${{ inputs.build-type }} RELEASE_VERSION: ${{ inputs.app-version }} - DEBUG: 'affine:*,napi:*' + DEBUG: affine:*,napi:* APP_NAME: affine MACOSX_DEPLOYMENT_TARGET: '11.6' - jobs: before-make: - if: ${{ inputs.desktop_macos || inputs.desktop_windows || inputs.desktop_linux }} runs-on: ubuntu-latest - environment: ${{ inputs.build-type }} steps: - uses: actions/checkout@v7 - name: Setup Version @@ -66,51 +39,19 @@ jobs: run: yarn affine @affine/electron generate-assets env: SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine' + SENTRY_PROJECT: affine SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} SENTRY_RELEASE: ${{ inputs.app-version }} RELEASE_VERSION: ${{ inputs.app-version }} - - name: Upload web artifact uses: actions/upload-artifact@v4 with: + overwrite: true name: desktop-web path: packages/frontend/apps/electron/resources/web-static - - windows-signer-gate: - if: ${{ inputs.desktop_windows }} - runs-on: ubuntu-latest - outputs: - signer_available: ${{ steps.check.outputs.signer_available }} - steps: - - uses: actions/checkout@v7 - - name: Check windows signer availability - id: check - run: node ./scripts/check-windows-signer.mjs - env: - AFFINE_SIGN_CLIENT_HASH: ${{ secrets.AFFINE_SIGN_CLIENT_HASH }} - AFFINE_SIGNER_ADDR: ${{ secrets.AFFINE_SIGNER_ADDR }} - AFFINE_SIGNER_TOKEN: ${{ secrets.AFFINE_SIGNER_TOKEN }} - AFFINE_SIGNER_TS_AUTH_KEY: ${{ secrets.AFFINE_SIGNER_TS_AUTH_KEY }} - BUILD_TYPE: ${{ inputs.build-type }} - REQUIRE_SIGNER: ${{ inputs.require-windows-signing }} - WINDOWS_SIGNER_PUBLIC_CERT_BASE64: ${{ secrets.WINDOWS_SIGNER_PUBLIC_CERT_BASE64 }} - - make-distribution-macos: - if: ${{ inputs.desktop_macos }} - strategy: - fail-fast: false - matrix: - spec: - - runner: macos-latest - platform: darwin - arch: x64 - target: x86_64-apple-darwin - - runner: macos-latest - platform: darwin - arch: arm64 - target: aarch64-apple-darwin + continue-on-error: ${{ !contains(inputs.matrix, '"required":true') }} + package: needs: before-make uses: ./.github/workflows/release-desktop-platform.yml secrets: inherit @@ -118,314 +59,4 @@ jobs: build_type: ${{ inputs.build-type }} app_version: ${{ inputs.app-version }} git_short_hash: ${{ inputs.git-short-hash }} - runner: ${{ matrix.spec.runner }} - platform: ${{ matrix.spec.platform }} - arch: ${{ matrix.spec.arch }} - target: ${{ matrix.spec.target }} - apple_codesign: true - - make-distribution-linux: - if: ${{ inputs.desktop_linux }} - strategy: - fail-fast: false - matrix: - spec: - - runner: ubuntu-latest - platform: linux - arch: x64 - target: x86_64-unknown-linux-gnu - needs: before-make - uses: ./.github/workflows/release-desktop-platform.yml - secrets: inherit - with: - build_type: ${{ inputs.build-type }} - app_version: ${{ inputs.app-version }} - git_short_hash: ${{ inputs.git-short-hash }} - runner: ${{ matrix.spec.runner }} - platform: ${{ matrix.spec.platform }} - arch: ${{ matrix.spec.arch }} - target: ${{ matrix.spec.target }} - install_linux_deps: true - - build-and-sign-windows: - if: ${{ inputs.desktop_windows && needs.windows-signer-gate.outputs.signer_available == 'true' }} - needs: - - before-make - - windows-signer-gate - strategy: - fail-fast: false - matrix: - spec: - - platform: win32 - arch: x64 - target: x86_64-pc-windows-msvc - - platform: win32 - arch: arm64 - target: aarch64-pc-windows-msvc - runs-on: windows-latest - env: - AFFINE_SIGNER_ADDR: ${{ secrets.AFFINE_SIGNER_ADDR }} - AFFINE_SIGNER_TOKEN: ${{ secrets.AFFINE_SIGNER_TOKEN }} - APP_NAME: affine - BUILD_TYPE: ${{ inputs.build-type }} - DEBUG: 'affine:*,napi:*' - RELEASE_VERSION: ${{ inputs.app-version }} - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine' - SENTRY_RELEASE: ${{ inputs.app-version }} - SKIP_GENERATE_ASSETS: 1 - TS_RS_EXPERIMENT: this_is_unstable_software - steps: - - uses: actions/checkout@v7 - - - name: Setup Version - uses: ./.github/actions/setup-version - with: - app-version: ${{ inputs.app-version }} - - - name: Setup Node.js - timeout-minutes: 10 - uses: ./.github/actions/setup-node - with: - extra-flags: workspaces focus @affine/electron @affine/monorepo @affine/nbstore @toeverything/infra - enableScripts: true - hard-link-nm: false - nmHoistingLimits: workspaces - env: - npm_config_arch: ${{ matrix.spec.arch }} - ELECTRON_INSTALL_ARCH: ${{ matrix.spec.arch }} - - - name: Build AFFiNE native - uses: ./.github/actions/build-rust - with: - target: ${{ matrix.spec.target }} - package: '@affine/native' - - - name: Download web artifact - uses: actions/download-artifact@v4 - with: - name: desktop-web - path: packages/frontend/apps/electron/resources/web-static - - - name: Build Desktop Layers - run: yarn affine @affine/electron build - - - name: Remove nbstore node_modules - shell: bash - run: | - rm -rf packages/frontend/apps/electron/node_modules/@affine/nbstore/node_modules/@blocksuite/affine/node_modules - rm -rf packages/frontend/apps/electron/node_modules/@affine/native/node_modules - - - name: Package windows app - run: yarn affine @affine/electron package --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }} - env: - HOIST_NODE_MODULES: 1 - NODE_OPTIONS: --max-old-space-size=14384 - SKIP_WEB_BUILD: 1 - - - name: Connect Tailscale - uses: tailscale/github-action@v4 - with: - authkey: ${{ secrets.AFFINE_SIGNER_TS_AUTH_KEY }} - hostname: affine-signer-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.spec.arch }} - - - name: Check signer connectivity - shell: pwsh - run: | - $Parts = "$env:AFFINE_SIGNER_ADDR".Split(':') - if ($Parts.Count -ne 2) { - throw "AFFINE_SIGNER_ADDR must be host:port, got $env:AFFINE_SIGNER_ADDR" - } - - $Result = Test-NetConnection -ComputerName $Parts[0] -Port ([int]$Parts[1]) - if (!$Result.TcpTestSucceeded) { - throw "Unable to connect to signer at $env:AFFINE_SIGNER_ADDR" - } - - - name: Download remote signer client - shell: pwsh - run: | - if (!$env:AFFINE_SIGN_CLIENT_HASH) { - throw 'AFFINE_SIGN_CLIENT_HASH is required.' - } - Invoke-WebRequest -Uri "https://cdn.affine.pro/sign-client/$env:AFFINE_SIGN_CLIENT_HASH" -OutFile affine-sign-client.exe - env: - AFFINE_SIGN_CLIENT_HASH: ${{ secrets.AFFINE_SIGN_CLIENT_HASH }} - - - name: Prepare public signing certificate - shell: pwsh - run: | - if (!$env:WINDOWS_SIGNER_PUBLIC_CERT_BASE64) { - throw 'WINDOWS_SIGNER_PUBLIC_CERT_BASE64 is required.' - } - [IO.File]::WriteAllBytes('windows-signer-public.cer', [Convert]::FromBase64String($env:WINDOWS_SIGNER_PUBLIC_CERT_BASE64)) - env: - WINDOWS_SIGNER_PUBLIC_CERT_BASE64: ${{ secrets.WINDOWS_SIGNER_PUBLIC_CERT_BASE64 }} - - - name: Resolve signtool - shell: pwsh - run: | - $Command = Get-Command signtool.exe -ErrorAction SilentlyContinue - if ($Command) { - "SIGNTOOL=$($Command.Source)" >> $env:GITHUB_ENV - Write-Host "Using signtool: $($Command.Source)" - exit 0 - } - - $KitsRoot = "${env:ProgramFiles(x86)}\Windows Kits\10\bin" - $Candidates = @() - if (Test-Path -LiteralPath $KitsRoot) { - $Candidates = Get-ChildItem -Path $KitsRoot -Recurse -Filter signtool.exe | - Where-Object { $_.FullName -match '\\x64\\signtool\.exe$' } | - Sort-Object FullName -Descending - } - - if ($Candidates.Count -eq 0) { - throw "Unable to find signtool.exe under PATH or $KitsRoot" - } - - "SIGNTOOL=$($Candidates[0].FullName)" >> $env:GITHUB_ENV - Write-Host "Using signtool: $($Candidates[0].FullName)" - - - name: Get packaged files to sign - id: packaged_files_to_sign - shell: pwsh - run: | - Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/apps/electron/out -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\apps\electron\out\', '') + '"' }) -join ' ') - "FILES_TO_BE_SIGNED=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT - echo $FILES_TO_BE_SIGNED - - - name: Sign packaged files - shell: pwsh - run: | - ./affine-sign-client.exe ` - --server "$env:AFFINE_SIGNER_ADDR" ` - --token "$env:AFFINE_SIGNER_TOKEN" ` - --workdir packages/frontend/apps/electron/out ` - --files '${{ steps.packaged_files_to_sign.outputs.FILES_TO_BE_SIGNED }}' ` - --cert windows-signer-public.cer ` - --plain-tcp - - - name: Make squirrel.windows installer - run: yarn affine @affine/electron make-squirrel --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }} - - - name: Make nsis.windows installer - run: yarn affine @affine/electron make-nsis --platform=${{ matrix.spec.platform }} --arch=${{ matrix.spec.arch }} - - - name: Get installer files to sign - id: installer_files_to_sign - shell: pwsh - run: | - Set-Variable -Name FILES_TO_BE_SIGNED -Value ((Get-ChildItem -Path packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make -Recurse -File | Where-Object { $_.Extension -in @(".exe", ".node", ".dll", ".msi") } | ForEach-Object { '"' + $_.FullName.Replace((Get-Location).Path + '\packages\frontend\apps\electron\out\${{ env.BUILD_TYPE }}\make\', '') + '"' }) -join ' ') - "FILES_TO_BE_SIGNED=$FILES_TO_BE_SIGNED" >> $env:GITHUB_OUTPUT - echo $FILES_TO_BE_SIGNED - - - name: Sign installer files - shell: pwsh - run: | - ./affine-sign-client.exe ` - --server "$env:AFFINE_SIGNER_ADDR" ` - --token "$env:AFFINE_SIGNER_TOKEN" ` - --workdir packages/frontend/apps/electron/out/${{ env.BUILD_TYPE }}/make ` - --files '${{ steps.installer_files_to_sign.outputs.FILES_TO_BE_SIGNED }}' ` - --cert windows-signer-public.cer ` - --plain-tcp - - - name: Save artifacts - run: | - mkdir -p builds - mv packages/frontend/apps/electron/out/*/make/zip/win32/${{ matrix.spec.arch }}/AFFiNE*-win32-${{ matrix.spec.arch }}-*.zip ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.zip - mv packages/frontend/apps/electron/out/*/make/squirrel.windows/${{ matrix.spec.arch }}/*.exe ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.exe - mv packages/frontend/apps/electron/out/*/make/nsis.windows/${{ matrix.spec.arch }}/*.exe ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.nsis.exe - - - uses: actions/attest-build-provenance@v4 - with: - subject-path: | - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.zip - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.exe - ./builds/affine-${{ env.RELEASE_VERSION }}-${{ env.BUILD_TYPE }}-windows-${{ matrix.spec.arch }}.nsis.exe - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: affine-${{ matrix.spec.platform }}-${{ matrix.spec.arch }}-builds - path: builds - - release: - if: >- - ${{ - always() && - (inputs.desktop_macos || inputs.desktop_linux || inputs.desktop_windows) && - needs.before-make.result == 'success' && - (!inputs.desktop_macos || needs.make-distribution-macos.result == 'success') && - (!inputs.desktop_linux || needs.make-distribution-linux.result == 'success') && - ( - !inputs.desktop_windows || - !inputs.require-windows-signing || - needs.build-and-sign-windows.result == 'success' - ) - }} - needs: - [ - before-make, - make-distribution-macos, - make-distribution-linux, - build-and-sign-windows, - ] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - name: Download Artifacts (macos-x64) - if: ${{ inputs.desktop_macos }} - uses: actions/download-artifact@v4 - with: - name: affine-darwin-x64-builds - path: ./release - - name: Download Artifacts (macos-arm64) - if: ${{ inputs.desktop_macos }} - uses: actions/download-artifact@v4 - with: - name: affine-darwin-arm64-builds - path: ./release - - name: Download Artifacts (windows-x64) - if: ${{ inputs.desktop_windows && needs.build-and-sign-windows.result == 'success' }} - uses: actions/download-artifact@v4 - with: - name: affine-win32-x64-builds - path: ./release - - name: Download Artifacts (windows-arm64) - if: ${{ inputs.desktop_windows && needs.build-and-sign-windows.result == 'success' }} - uses: actions/download-artifact@v4 - with: - name: affine-win32-arm64-builds - path: ./release - - name: Download Artifacts (linux-x64) - if: ${{ inputs.desktop_linux }} - uses: actions/download-artifact@v4 - with: - name: affine-linux-x64-builds - path: ./release - - uses: actions/setup-node@v7 - with: - node-version-file: '.nvmrc' - - name: Copy Selfhost Release Files - run: | - cp ./.docker/selfhost/compose.yml ./release/docker-compose.yml - cp ./.docker/selfhost/config.json.example ./release/config.json.example - cp ./.docker/selfhost/schema.json ./release/config.schema.json - - name: Generate Release yml - run: | - node ./scripts/generate-release-yml.mjs - env: - RELEASE_VERSION: ${{ env.RELEASE_VERSION }} - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - name: ${{ env.RELEASE_VERSION }} - draft: ${{ inputs.build-type == 'stable' }} - prerelease: ${{ inputs.build-type != 'stable' }} - tag_name: v${{ env.RELEASE_VERSION}} - files: ./release/* + matrix: ${{ inputs.matrix }} diff --git a/.github/workflows/release-mobile.yml b/.github/workflows/release-mobile.yml index 5f2db2b9df..8cd1de7138 100644 --- a/.github/workflows/release-mobile.yml +++ b/.github/workflows/release-mobile.yml @@ -1,6 +1,5 @@ -name: Release Mobile - -on: +name: Build Mobile Candidates +'on': workflow_call: inputs: app-version: @@ -12,19 +11,16 @@ on: build-type: type: string required: true - ios-app-version: + matrix: type: string - required: false - + required: true env: BUILD_TYPE: ${{ inputs.build-type }} DEBUG: napi:* KEYCHAIN_NAME: ${{ github.workspace }}/signing_temp - jobs: - build-ios-web: - runs-on: ubuntu-latest - environment: ${{ inputs.build-type }} + web: + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v7 - name: Setup Version @@ -36,11 +32,11 @@ jobs: - name: Setup @sentry/cli uses: ./.github/actions/setup-sentry - name: Build Mobile - run: yarn affine @affine/ios build + run: yarn affine bundle -p ${{ matrix.platform }} env: - PUBLIC_PATH: '/' + PUBLIC_PATH: / SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine' + SENTRY_PROJECT: affine SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} SENTRY_RELEASE: ${{ inputs.app-version }} @@ -48,181 +44,13 @@ jobs: - name: Upload ios artifact uses: actions/upload-artifact@v4 with: - name: ios - path: packages/frontend/apps/ios/dist - - build-android-web: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - name: Setup Version - uses: ./.github/actions/setup-version - with: - app-version: ${{ inputs.app-version }} - - name: Setup Node.js - uses: ./.github/actions/setup-node - - name: Setup @sentry/cli - uses: ./.github/actions/setup-sentry - - name: Build Mobile - run: yarn affine @affine/android build - env: - PUBLIC_PATH: '/' - SENTRY_ORG: ${{ secrets.SENTRY_ORG }} - SENTRY_PROJECT: 'affine' - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - SENTRY_RELEASE: ${{ inputs.app-version }} - - name: Upload android artifact - uses: actions/upload-artifact@v4 - with: - name: android - path: packages/frontend/apps/android/dist - - ios: - runs-on: 'macos-15' - needs: - - build-ios-web - steps: - - uses: actions/checkout@v7 - - name: Setup Version - uses: ./.github/actions/setup-version - with: - app-version: ${{ inputs.app-version }} - ios-app-version: ${{ inputs.ios-app-version }} - - name: 'Update Code Sign Identity' - shell: bash - run: ./packages/frontend/apps/ios/update_code_sign_identity.sh - - name: Download mobile artifact - uses: actions/download-artifact@v4 - with: - name: ios - path: packages/frontend/apps/ios/dist - - name: Setup Node.js - uses: ./.github/actions/setup-node - timeout-minutes: 10 - with: - extra-flags: workspaces focus @affine/ios - playwright-install: false - electron-install: false - hard-link-nm: false - enableScripts: false - - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: 26.2 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.3' - - name: Install Swiftformat - run: brew install swiftformat - - name: Cap sync - run: yarn workspace @affine/ios sync - - name: Signing By Apple Developer ID - uses: apple-actions/import-codesign-certs@v7 - id: import-codesign-certs - with: - p12-file-base64: ${{ secrets.CERTIFICATES_P12_MOBILE }} - p12-password: ${{ secrets.CERTIFICATES_P12_PASSWORD_MOBILE }} - - name: Setup Rust - uses: ./.github/actions/build-rust - with: - target: 'aarch64-apple-ios' - package: 'affine_mobile_native' - no-build: 'true' - - name: Testflight - working-directory: packages/frontend/apps/ios/App - run: | - printf '%s' "$BUILD_PROVISION_PROFILE" | base64 --decode -o "$PP_PATH" - mkdir -p "$HOME/Library/MobileDevice/Provisioning Profiles" - cp "$PP_PATH" "$HOME/Library/MobileDevice/Provisioning Profiles" - bundle install - bundle exec fastlane beta - env: - BUNDLE_PATH: vendor/bundle - BUILD_TARGET: distribution - BUILD_PROVISION_PROFILE: ${{ secrets.BUILD_PROVISION_PROFILE }} - PP_PATH: ${{ runner.temp }}/build_pp.mobileprovision - APPLE_STORE_CONNECT_API_KEY_ID: ${{ secrets.APPLE_STORE_CONNECT_API_KEY_ID }} - APPLE_STORE_CONNECT_API_ISSUER_ID: ${{ secrets.APPLE_STORE_CONNECT_API_ISSUER_ID }} - APPLE_STORE_CONNECT_API_KEY: ${{ secrets.APPLE_STORE_CONNECT_API_KEY }} - - android: - runs-on: ubuntu-latest - permissions: - id-token: 'write' - needs: - - build-android-web - steps: - - uses: actions/checkout@v7 - - name: Setup Version - uses: ./.github/actions/setup-version - with: - app-version: ${{ inputs.app-version }} - - name: Download mobile artifact - uses: actions/download-artifact@v4 - with: - name: android - path: packages/frontend/apps/android/dist - - name: Load Google Service file - env: - DATA: ${{ secrets.FIREBASE_ANDROID_GOOGLE_SERVICE_JSON }} - run: | - set -euo pipefail - printf '%s' "$DATA" | base64 -di > packages/frontend/apps/android/App/app/google-services.json - - name: Setup Node.js - uses: ./.github/actions/setup-node - timeout-minutes: 10 - with: - extra-flags: workspaces focus @affine/monorepo @affine-tools/cli @affine/android @affine/playstore-auto-bump - playwright-install: false - electron-install: false - hard-link-nm: false - enableScripts: false - - name: Setup Rust - uses: ./.github/actions/build-rust - with: - target: 'aarch64-linux-android' - package: 'affine_mobile_native' - no-build: 'true' - - name: Cap sync - run: yarn workspace @affine/android cap sync - - uses: actions/setup-python@v7 - with: - python-version: '3.13' - - name: Auth gcloud - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: 'projects/${{ secrets.GCP_PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions-helm-deploy' - service_account: '${{ secrets.GCP_HELM_DEPLOY_SERVICE_ACCOUNT }}' - token_format: 'access_token' - project_id: '${{ secrets.GCP_PROJECT_ID }}' - access_token_scopes: 'https://www.googleapis.com/auth/androidpublisher' - - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '21' - cache: 'gradle' - - name: Auto increment version code - id: bump - run: yarn affine @affine/playstore-auto-bump bump - env: - GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.auth.outputs.credentials_file_path }} - - name: Build - run: | - echo -n "${{ env.AFFINE_ANDROID_SIGN_KEYSTORE }}" | base64 --decode > packages/frontend/apps/android/affine.keystore - yarn workspace @affine/android cap build android --flavor ${{ env.BUILD_TYPE }} --androidreleasetype AAB - env: - AFFINE_ANDROID_KEYSTORE_PASSWORD: ${{ secrets.AFFINE_ANDROID_KEYSTORE_PASSWORD }} - AFFINE_ANDROID_KEYSTORE_ALIAS_PASSWORD: ${{ secrets.AFFINE_ANDROID_KEYSTORE_ALIAS_PASSWORD }} - AFFINE_ANDROID_SIGN_KEYSTORE: ${{ secrets.AFFINE_ANDROID_SIGN_KEYSTORE }} - VERSION_NAME: ${{ inputs.app-version }} - - name: Upload to Google Play - uses: r0adkll/upload-google-play@v1 - with: - serviceAccountJson: ${{ steps.auth.outputs.credentials_file_path }} - packageName: app.affine.pro - releaseName: ${{ inputs.app-version }} - releaseFiles: packages/frontend/apps/android/App/app/build/outputs/bundle/${{ env.BUILD_TYPE }}Release/app-${{ env.BUILD_TYPE }}-release-signed.aab - track: internal - status: draft - existingEditId: ${{ steps.bump.outputs.EDIT_ID }} + overwrite: true + name: candidate-mobile-${{ matrix.platform }} + path: packages/frontend/apps/${{ matrix.platform }}/dist + if-no-files-found: error + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + continue-on-error: ${{ !matrix.required }} +permissions: + contents: read diff --git a/.github/workflows/release-submit.yml b/.github/workflows/release-submit.yml new file mode 100644 index 0000000000..3189409562 --- /dev/null +++ b/.github/workflows/release-submit.yml @@ -0,0 +1,43 @@ +name: Submit Release Candidate +on: + workflow_run: + workflows: [Release Candidates] + types: [completed] +permissions: + contents: read + actions: read +jobs: + submit: + if: >- + ${{ github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == 'toeverything/AFFiNE' && + contains(fromJSON('["workflow_dispatch","schedule"]'), github.event.workflow_run.event) }} + runs-on: ubuntu-latest + steps: + - name: Find completed candidate + id: candidate + uses: actions/github-script@v9 + with: + script: | + const run = context.payload.workflow_run; + if (run.path.split('@')[0] !== '.github/workflows/release.yml') throw new Error('Unexpected candidate workflow'); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts,{...context.repo,run_id:run.id}); + core.setOutput('present', String(artifacts.some(a => a.name === 'release-candidate' && !a.expired))); + - uses: actions/create-github-app-token@v2 + if: ${{ steps.candidate.outputs.present == 'true' }} + id: token + with: + app-id: ${{ secrets.AFFINE_RELEASE_APP_ID }} + private-key: ${{ secrets.AFFINE_RELEASE_APP_PRIVATE_KEY }} + owner: toeverything + repositories: affine-native + - name: Submit candidate identity + if: ${{ steps.candidate.outputs.present == 'true' }} + uses: actions/github-script@v9 + with: + github-token: ${{ steps.token.outputs.token }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner:'toeverything',repo:'affine-native',workflow_id:'affine-release.yml',ref:'master', + inputs:{'candidate-run-id':String(context.payload.workflow_run.id),command:'apply'}, + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 53005572a6..730d03f1ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,211 +1,211 @@ -name: Release +name: Release Candidates on: schedule: - cron: '0 9 * * *' - workflow_dispatch: inputs: - web: - description: 'Release Web?' + targets: + description: 'Targets (comma-separated; append ? for optional platforms)' required: true - type: boolean - default: false - desktop_macos: - description: 'Desktop - macOS' - required: true - type: boolean - default: false - desktop_windows: - description: 'Desktop - Windows' - required: true - type: boolean - default: false - desktop_linux: - description: 'Desktop - Linux' - required: true - type: boolean - default: false - mobile: - description: 'Release Mobile?' - required: true - type: boolean - default: false - ios-app-version: - description: 'iOS App Store Version (Optional, use tag version if empty)' - required: false + default: 'server,desktop-macos,desktop-windows,desktop-linux?' type: string permissions: - contents: write - pull-requests: write - actions: write - id-token: write + contents: read + actions: read packages: write - security-events: write - attestations: write - issues: write jobs: prepare: - name: Prepare + permissions: + contents: read + actions: read + packages: read runs-on: ubuntu-latest outputs: - APP_VERSION: ${{ steps.prepare.outputs.APP_VERSION }} - GIT_SHORT_HASH: ${{ steps.prepare.outputs.GIT_SHORT_HASH }} - BUILD_TYPE: ${{ steps.prepare.outputs.BUILD_TYPE }} + version: ${{ steps.plan.outputs.APP_VERSION }} + sha: ${{ steps.plan.outputs.GIT_SHORT_HASH }} + channel: ${{ steps.plan.outputs.BUILD_TYPE }} + server: ${{ steps.plan.outputs.SERVER }} + skip: ${{ steps.plan.outputs.SKIP }} + desktop: ${{ steps.plan.outputs.DESKTOP_MATRIX }} + mobile: ${{ steps.plan.outputs.MOBILE_MATRIX }} steps: - uses: actions/checkout@v7 - - name: Prepare Release - id: prepare - uses: ./.github/actions/prepare-release - - canary-gate: - name: Canary Gate - runs-on: ubuntu-latest - needs: - - prepare - outputs: - SHOULD_RELEASE: ${{ steps.decide.outputs.SHOULD_RELEASE }} - LAST_CANARY_TAG: ${{ steps.decide.outputs.LAST_CANARY_TAG }} - LAST_CANARY_SHA: ${{ steps.decide.outputs.LAST_CANARY_SHA }} - steps: - - name: Decide whether to release - id: decide + - name: Preserve request creation time + id: run uses: actions/github-script@v9 with: script: | - const buildType = '${{ needs.prepare.outputs.BUILD_TYPE }}' - if (buildType !== 'canary') { - core.setOutput('SHOULD_RELEASE', 'true') - return - } - - const owner = context.repo.owner - const repo = context.repo.repo - const currentSha = context.sha - const canaryTagRe = /^v\d+\.\d+\.\d+-canary\.[0-9a-f]+$/i - - let page = 1 - const perPage = 100 - let lastCanary = null - - while (!lastCanary && page <= 10) { - const { data } = await github.rest.repos.listTags({ - owner, - repo, - per_page: perPage, - page, - }) - - for (const tag of data) { - if (canaryTagRe.test(tag.name)) { - lastCanary = tag - break - } + const run = await github.rest.actions.getWorkflowRun({...context.repo, run_id: context.runId}); + core.setOutput('created-at', run.data.created_at); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts,{...context.repo,run_id:context.runId}); + if (artifacts.some(a => a.name === 'release-candidate')) { + for (let attempt = 1; attempt < Number(process.env.GITHUB_RUN_ATTEMPT); attempt++) { + const previous = await github.rest.actions.getWorkflowRunAttempt({...context.repo,run_id:context.runId,attempt_number:attempt}); + if (previous.data.conclusion === 'success') throw new Error('This run already submitted an immutable candidate. Recover its native request or start a new release run.'); } - - if (data.length < perPage) break - page++ } - - if (!lastCanary) { - core.warning('No canary tags found; proceeding with canary release.') - core.setOutput('SHOULD_RELEASE', 'true') - return + - name: Read completed release evidence + uses: actions/github-script@v9 + with: + script: | + const fs = require('node:fs'); + const releases = await github.paginate(github.rest.repos.listReleases, {...context.repo, per_page: 100}); + for (const release of releases) { + const asset = release.assets.find(asset => asset.name === 'affine-release-completed.json'); + if (!asset) continue; + const response = await github.rest.repos.getReleaseAsset({...context.repo, asset_id: asset.id, headers: {accept:'application/octet-stream'}}); + const previous = JSON.parse(Buffer.from(response.data).toString('utf8')); + if (previous.request.channel !== 'canary') continue; + fs.writeFileSync('release-previous.json', JSON.stringify(previous)); + break; } + - uses: ./.github/actions/prepare-release + id: plan + with: + targets: ${{ inputs.targets || 'server,desktop-macos,desktop-windows,desktop-linux?' }} + tool-image: ${{ vars.AFFINE_RELEASE_TOOL_IMAGE || 'ghcr.io/toeverything/affine-release:latest' }} + created-at: ${{ steps.run.outputs.created-at }} + - uses: actions/upload-artifact@v4 + with: + overwrite: true + name: release-plan + path: release-plan.json + if-no-files-found: error - core.setOutput('LAST_CANARY_TAG', lastCanary.name) - core.setOutput('LAST_CANARY_SHA', lastCanary.commit.sha) - - const shouldRelease = lastCanary.commit.sha !== currentSha - core.info(`Latest canary tag ${lastCanary.name} -> ${lastCanary.commit.sha}; current ${currentSha}; should_release=${shouldRelease}`) - core.setOutput('SHOULD_RELEASE', shouldRelease ? 'true' : 'false') - - cloud: - name: Release Cloud - if: ${{ inputs.web || github.event_name != 'workflow_dispatch' }} - needs: - - prepare - uses: ./.github/workflows/release-cloud.yml + server: + needs: prepare + if: ${{ needs.prepare.outputs.skip != 'true' && needs.prepare.outputs.server == 'true' }} + uses: ./.github/workflows/build-images.yml secrets: inherit with: - build-type: ${{ needs.prepare.outputs.BUILD_TYPE }} - app-version: ${{ needs.prepare.outputs.APP_VERSION }} - git-short-hash: ${{ needs.prepare.outputs.GIT_SHORT_HASH }} - - image: - name: Release Docker Image - if: ${{ needs.canary-gate.outputs.SHOULD_RELEASE == 'true' }} - runs-on: ubuntu-latest - needs: - - prepare - - canary-gate - - cloud - steps: - - uses: trstringer/manual-approval@v1 - if: ${{ needs.prepare.outputs.BUILD_TYPE == 'stable' }} - name: Wait for approval - with: - secret: ${{ secrets.GITHUB_TOKEN }} - approvers: darkskygit - minimum-approvals: 1 - fail-on-denial: true - issue-title: Please confirm to release docker image - issue-body: | - Env: ${{ needs.prepare.outputs.BUILD_TYPE }} - Candidate: ghcr.io/toeverything/affine:${{ needs.prepare.outputs.BUILD_TYPE }}-${{ needs.prepare.outputs.GIT_SHORT_HASH }} - Tag: ghcr.io/toeverything/affine:${{ needs.prepare.outputs.BUILD_TYPE }} - - > comment with "approve", "approved", "lgtm", "yes" to approve - > comment with "deny", "denied", "no" to deny - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - logout: false - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Tag Image - run: | - docker buildx imagetools create --tag ghcr.io/toeverything/affine:${{needs.prepare.outputs.BUILD_TYPE}} ghcr.io/toeverything/affine:${{needs.prepare.outputs.BUILD_TYPE}}-${{needs.prepare.outputs.GIT_SHORT_HASH}} - docker buildx imagetools create --tag ghcr.io/toeverything/affine:${{needs.prepare.outputs.APP_VERSION}} ghcr.io/toeverything/affine:${{needs.prepare.outputs.BUILD_TYPE}}-${{needs.prepare.outputs.GIT_SHORT_HASH}} + app-version: ${{ needs.prepare.outputs.version }} + git-short-hash: ${{ needs.prepare.outputs.sha }} + build-type: ${{ needs.prepare.outputs.channel }} desktop: - name: Release Desktop - if: >- - ${{ - (github.event_name != 'workflow_dispatch' && needs.canary-gate.outputs.SHOULD_RELEASE == 'true') || - inputs.desktop_macos || - inputs.desktop_windows || - inputs.desktop_linux - }} - needs: - - prepare - - canary-gate + needs: prepare + if: ${{ needs.prepare.outputs.skip != 'true' && fromJSON(needs.prepare.outputs.desktop).include[0] != null }} uses: ./.github/workflows/release-desktop.yml secrets: inherit with: - build-type: ${{ needs.prepare.outputs.BUILD_TYPE }} - app-version: ${{ needs.prepare.outputs.APP_VERSION }} - git-short-hash: ${{ needs.prepare.outputs.GIT_SHORT_HASH }} - desktop_macos: ${{ github.event_name != 'workflow_dispatch' || inputs.desktop_macos }} - desktop_windows: ${{ github.event_name != 'workflow_dispatch' || inputs.desktop_windows }} - desktop_linux: ${{ github.event_name != 'workflow_dispatch' || inputs.desktop_linux }} - require-windows-signing: ${{ needs.prepare.outputs.BUILD_TYPE == 'stable' || (github.event_name == 'workflow_dispatch' && inputs.desktop_windows) }} + app-version: ${{ needs.prepare.outputs.version }} + git-short-hash: ${{ needs.prepare.outputs.sha }} + build-type: ${{ needs.prepare.outputs.channel }} + matrix: ${{ needs.prepare.outputs.desktop }} mobile: - name: Release Mobile - if: ${{ inputs.mobile }} - needs: - - prepare + needs: prepare + if: ${{ needs.prepare.outputs.skip != 'true' && fromJSON(needs.prepare.outputs.mobile).include[0] != null }} uses: ./.github/workflows/release-mobile.yml secrets: inherit with: - build-type: ${{ needs.prepare.outputs.BUILD_TYPE }} - app-version: ${{ needs.prepare.outputs.APP_VERSION }} - git-short-hash: ${{ needs.prepare.outputs.GIT_SHORT_HASH }} - ios-app-version: ${{ inputs.ios-app-version }} + app-version: ${{ needs.prepare.outputs.version }} + git-short-hash: ${{ needs.prepare.outputs.sha }} + build-type: ${{ needs.prepare.outputs.channel }} + matrix: ${{ needs.prepare.outputs.mobile }} + + candidate: + permissions: + contents: read + actions: write + needs: [prepare, server, desktop, mobile] + if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.skip != 'true' }} + runs-on: ubuntu-latest + steps: + - name: Prepare candidate rerun artifacts + uses: actions/github-script@v9 + with: + script: | + if (Number(process.env.GITHUB_RUN_ATTEMPT) === 1) return; + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts,{...context.repo,run_id:context.runId}); + if (artifacts.some(a => a.name === 'release-candidate')) { + for (let attempt = 1; attempt < Number(process.env.GITHUB_RUN_ATTEMPT); attempt++) { + const previous = await github.rest.actions.getWorkflowRunAttempt({...context.repo,run_id:context.runId,attempt_number:attempt}); + if (previous.data.conclusion === 'success') throw new Error('Submitted candidates are immutable. Recover the native request or start a new run.'); + } + } + for (const artifact of artifacts.filter(a => ['candidate-desktop-macos','candidate-desktop-windows','candidate-desktop-linux'].includes(a.name))) { + await github.rest.actions.deleteArtifact({...context.repo,artifact_id:artifact.id}); + } + - uses: actions/download-artifact@v4 + with: + name: release-plan + - name: Combine desktop architecture candidates + uses: actions/upload-artifact/merge@v4 + continue-on-error: true + with: + name: candidate-desktop-macos + pattern: candidate-desktop-darwin-* + if: ${{ contains(needs.prepare.outputs.desktop, 'desktop-macos') }} + - name: Combine Windows architecture candidates + uses: actions/upload-artifact/merge@v4 + continue-on-error: true + with: + name: candidate-desktop-windows + pattern: candidate-desktop-win32-* + if: ${{ contains(needs.prepare.outputs.desktop, 'desktop-windows') }} + - name: Combine Linux architecture candidates + uses: actions/upload-artifact/merge@v4 + continue-on-error: true + with: + name: candidate-desktop-linux + pattern: candidate-desktop-linux-* + if: ${{ contains(needs.prepare.outputs.desktop, 'desktop-linux') }} + - name: Collect static assets + if: ${{ needs.server.result == 'success' }} + uses: actions/download-artifact@v4 + with: + pattern: '{web,admin,mobile}' + merge-multiple: true + path: static-assets + - name: Save static assets candidate + if: ${{ needs.server.result == 'success' }} + uses: actions/upload-artifact@v4 + with: + overwrite: true + name: static-assets + path: | + static-assets/** + !static-assets/**/*.html + if-no-files-found: error + - name: Record candidate identities + uses: actions/github-script@v9 + env: + IMAGE: ${{ needs.server.outputs.image }} + with: + script: | + const fs = require('node:fs'); + const plan = JSON.parse(fs.readFileSync('release-plan.json', 'utf8')); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, + }); + const candidate = { + repository: `${context.repo.owner}/${context.repo.repo}`, + runId: context.runId, workflow: '.github/workflows/release.yml', + event: context.eventName, ref: context.ref, sha: context.sha, + image: process.env.IMAGE || '', artifacts: {}, + }; + for (const target of plan.targets) { + if (target.name === 'server') continue; + const artifact = artifacts.find(a => a.name === `candidate-${target.name}` && !a.expired); + if (artifact) candidate.artifacts[target.name] = { + id: artifact.id, runId: context.runId, repository: candidate.repository, + name: artifact.name, stage: 'unsigned', + }; + } + const statics = artifacts.find(a => a.name === 'static-assets' && !a.expired); + if (statics) candidate.artifacts['static-assets'] = { + id: statics.id, runId: context.runId, repository: candidate.repository, + name: statics.name, stage: 'ready', + }; + plan.candidate = candidate; + fs.writeFileSync('request.json', JSON.stringify(plan, null, 2)); + - uses: actions/upload-artifact@v4 + with: + overwrite: true + name: release-candidate + path: request.json + if-no-files-found: error diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index f0936f1d40..9742b25c24 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -115,7 +115,7 @@ export declare class BackendRuntime { revokeWorkspaceInviteLink(workspaceId: string): Promise cleanupExpiredRuntimeStates(limit: number): Promise cleanupExpiredStorageReservationsV1(limit: number): Promise - listManagedWorkspaceBlobsV1(actorUserId: string, workspaceId: string): Promise> + listManagedWorkspaceBlobsV1(actorUserId: string, workspaceId: string, limit: number, after?: string | undefined | null): Promise> manageWorkspaceBlobV1(input: RuntimeBlobManagementInput): Promise releaseManagedWorkspaceBlobsV1(actorUserId: string, workspaceId: string, limit: number): Promise finalizeStorageReservationV1(input: RuntimeStorageReservationMutation): Promise @@ -612,6 +612,8 @@ export interface DocumentEmbeddingProjectionInput { docId: string revision: string sourceHash: string + skipReason?: string + rootRevision?: string units: Array deleted?: boolean } @@ -989,6 +991,11 @@ export interface NativeWorkspaceDocContent { avatarKey: string } +export interface NativeWorkspaceRootProjection { + complete: boolean + docIds: Array +} + export interface ParsedDoc { name: string chunks: Array @@ -1031,6 +1038,8 @@ export declare function projectDocCanvasFromBinary(docBin: Buffer, docId: string export declare function projectDocSearchFromBinary(docBin: Buffer, docId: string, revision: string): NativeDocumentSearchProjection +export declare function projectWorkspaceRootFromBinary(docBin: Buffer): NativeWorkspaceRootProjection + export type PromptBuiltin = 'Date'| 'Language'| 'Timezone'| diff --git a/packages/backend/native/src/doc.rs b/packages/backend/native/src/doc.rs index 5fac33c58e..21885b989c 100644 --- a/packages/backend/native/src/doc.rs +++ b/packages/backend/native/src/doc.rs @@ -287,6 +287,24 @@ pub fn project_doc_canvas_from_binary( Ok(projection.into()) } +#[napi(object)] +pub struct NativeWorkspaceRootProjection { + pub complete: bool, + pub doc_ids: Vec, +} + +#[napi] +pub fn project_workspace_root_from_binary(doc_bin: Buffer) -> Result { + let projection = map_napi_err( + doc_loader::project_workspace_root(doc_bin.into(), true), + Status::GenericFailure, + )?; + Ok(NativeWorkspaceRootProjection { + complete: projection.complete, + doc_ids: projection.doc_ids, + }) +} + #[napi] pub fn project_doc_search_from_binary( doc_bin: Buffer, diff --git a/packages/backend/native/src/runtime/backend_runtime/auth_session/keyring.rs b/packages/backend/native/src/runtime/backend_runtime/auth_session/keyring.rs index 89c350bf82..fa127ee883 100644 --- a/packages/backend/native/src/runtime/backend_runtime/auth_session/keyring.rs +++ b/packages/backend/native/src/runtime/backend_runtime/auth_session/keyring.rs @@ -18,7 +18,8 @@ struct StoredKey { id: String, secret: String, status: String, - // TODO(0.27.5): Remove old-reader serialization constraints after 0.27.4 exits. + // TODO(0.27.5): Remove old-reader serialization constraints after 0.27.4 + // exits. #[serde(skip_serializing_if = "Option::is_none")] created_at: Option>, source: String, diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/source.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/source.rs index 95b8fb85b4..84c57606b9 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/source.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/source.rs @@ -66,6 +66,18 @@ pub(super) async fn sync_documents( .await .map_err(|error| RuntimeError::database("upsert document embedding source failed", error))?; queue_active_projection(&mut transaction, workspace_id, source_id, priority).await?; + if let Some(reason) = &document.skip_reason { + sqlx::query( + "UPDATE embedding_projections SET \ + status='failed',last_error_code=$2,last_error_detail=NULL,lease_owner=NULL,lease_until=NULL,\ + lease_token=lease_token+1,updated_at=now() WHERE source_id=$1", + ) + .bind(source_id) + .bind(reason) + .execute(&mut *transaction) + .await + .map_err(|error| RuntimeError::database("record skipped document projection failed", error))?; + } } if let Some(live_doc_ids) = live_doc_ids { reconcile_document_sources(&mut transaction, workspace_id, &live_doc_ids).await?; diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs index b6ab3603a2..553b1f8fbb 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/store.rs @@ -473,6 +473,8 @@ mod tests { ); let first_token = commit_token(&pool, ¤t, &[chunk("first")]).await.unwrap(); let requested = [crate::runtime::types::DocumentEmbeddingProjectionInput { + skip_reason: None, + root_revision: None, doc_id: "doc".to_string(), revision: "content-1".to_string(), source_hash: "descriptor".to_string(), @@ -580,6 +582,42 @@ mod tests { assert_eq!(result.chunks, 1); assert_eq!(result.indexes, 1); + let skipped = [crate::runtime::types::DocumentEmbeddingProjectionInput { + revision: "content-3".to_string(), + skip_reason: Some("document_contains_nul".to_string()), + ..updated[0].clone() + }]; + super::super::source::sync_documents(&pool, &workspace_id, &skipped, false, i32::MAX) + .await + .unwrap(); + let skipped_state: (String, Option, String) = sqlx::query_as( + "SELECT projection.status,projection.last_error_code,source.content_revision FROM embedding_projections \ + projection JOIN embedding_sources source ON source.id=projection.source_id WHERE source.id=$1", + ) + .bind(source_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + skipped_state, + ( + "failed".into(), + Some("document_contains_nul".into()), + "content-3".into() + ) + ); + assert!(claim_projection(&pool, "skip-check").await.unwrap().is_none()); + let repaired = [crate::runtime::types::DocumentEmbeddingProjectionInput { + revision: "content-4".to_string(), + ..updated[0].clone() + }]; + super::super::source::sync_documents(&pool, &workspace_id, &repaired, false, i32::MAX) + .await + .unwrap(); + let repaired_claim = claim_projection(&pool, "repaired-worker").await.unwrap().unwrap(); + assert_eq!(repaired_claim.content_revision, "content-4"); + assert_eq!(repaired_claim.source_id, source_id); + sqlx::query("DELETE FROM embedding_workspace_states WHERE workspace_id=$1") .bind(&workspace_id) .execute(&pool) diff --git a/packages/backend/native/src/runtime/backend_runtime/embedding/worker.rs b/packages/backend/native/src/runtime/backend_runtime/embedding/worker.rs index 46e0c89713..b6f0c6ddb0 100644 --- a/packages/backend/native/src/runtime/backend_runtime/embedding/worker.rs +++ b/packages/backend/native/src/runtime/backend_runtime/embedding/worker.rs @@ -143,6 +143,9 @@ async fn materialize( .ok_or_else(|| failure("document_projection_missing", None))?, ) .map_err(|_| failure("document_projection_invalid", None))?; + if let Some(reason) = projection.skip_reason { + return Err(failure("document_projection_skipped", Some(reason))); + } let mut contents = Vec::with_capacity(projection.units.len()); let mut locators = Vec::with_capacity(projection.units.len()); for unit in projection.units { diff --git a/packages/backend/native/src/runtime/backend_runtime/payment/financial_reconcile.rs b/packages/backend/native/src/runtime/backend_runtime/payment/financial_reconcile.rs index ee19051e0d..b864a786df 100644 --- a/packages/backend/native/src/runtime/backend_runtime/payment/financial_reconcile.rs +++ b/packages/backend/native/src/runtime/backend_runtime/payment/financial_reconcile.rs @@ -49,12 +49,12 @@ async fn reconcile_financial( ) -> RuntimeResult { let namespace = &candidate.namespace; let mut resources = vec![PaymentScope::financial( - &namespace, + namespace, &candidate.object_kind, &candidate.external_id, )?]; if let Some(source_id) = candidate.source_id.as_deref() { - resources.push(PaymentScope::source(&namespace, source_id)?); + resources.push(PaymentScope::source(namespace, source_id)?); } let connection = PaymentConnection::try_acquire(&runtime.pool, resources) .await? diff --git a/packages/backend/native/src/runtime/backend_runtime/strict_quota/management.rs b/packages/backend/native/src/runtime/backend_runtime/strict_quota/management.rs index 251d5c1766..0b7b9af269 100644 --- a/packages/backend/native/src/runtime/backend_runtime/strict_quota/management.rs +++ b/packages/backend/native/src/runtime/backend_runtime/strict_quota/management.rs @@ -11,7 +11,7 @@ use crate::runtime::{ types::{RuntimeBlobManagementInput, RuntimeManagedBlob}, }; -const MAX_MANAGED_BLOB_INVENTORY: i64 = 1_000; +const MAX_MANAGED_BLOB_PAGE_SIZE: i64 = 1_000; #[napi_derive::napi] impl BackendRuntime { @@ -20,7 +20,12 @@ impl BackendRuntime { &self, actor_user_id: String, workspace_id: String, + limit: i64, + after: Option, ) -> Result> { + if !(1..=MAX_MANAGED_BLOB_PAGE_SIZE).contains(&limit) { + return Err(RuntimeError::invalid_input("invalid_blob_page_size").into()); + } let pool = self.pool().await?; let deployment = self.config()?.deployment; let authorizer = PermissionAuthorizer::with_telemetry(pool.clone(), deployment, self.permission_telemetry.clone()); @@ -31,16 +36,14 @@ impl BackendRuntime { assert_manage_allowed(&mut tx, &authorizer, &workspace_id, &actor_user_id, false).await?; let rows = sqlx::query( "SELECT key,mime,size,created_at FROM blobs WHERE workspace_id=$1 AND status='completed' AND deleted_at IS NULL \ - ORDER BY key LIMIT $2", + AND ($3::text IS NULL OR key > $3) ORDER BY key LIMIT $2", ) .bind(&workspace_id) - .bind(MAX_MANAGED_BLOB_INVENTORY + 1) + .bind(limit) + .bind(after) .fetch_all(&mut *tx) .await .map_err(|error| RuntimeError::database("list managed workspace blobs", error))?; - if rows.len() > usize::try_from(MAX_MANAGED_BLOB_INVENTORY).expect("managed blob cap fits usize") { - return Err(RuntimeError::invalid_state("managed_blob_inventory_too_large").into()); - } tx.commit() .await .map_err(|error| RuntimeError::database("commit managed blob inventory", error))?; diff --git a/packages/backend/native/src/runtime/backend_runtime/tests.rs b/packages/backend/native/src/runtime/backend_runtime/tests.rs index 2bcf7bef67..9a2fe356cf 100644 --- a/packages/backend/native/src/runtime/backend_runtime/tests.rs +++ b/packages/backend/native/src/runtime/backend_runtime/tests.rs @@ -1663,12 +1663,12 @@ async fn strict_blob_management_authorizes_inventory_and_denies_before_object_cl assert!( runtime - .list_managed_workspace_blobs_v1(member_id.into(), workspace_id.into()) + .list_managed_workspace_blobs_v1(member_id.into(), workspace_id.into(), 1000, None) .await .is_err() ); let inventory = runtime - .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into()) + .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into(), 1000, None) .await .unwrap(); assert_eq!(inventory.len(), 1); @@ -1681,12 +1681,28 @@ async fn strict_blob_management_authorizes_inventory_and_denies_before_object_cl .execute(&pool) .await .unwrap(); - assert!( - runtime - .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into()) - .await - .is_err() - ); + let first_page = runtime + .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into(), 1000, None) + .await + .unwrap(); + let last_page = runtime + .list_managed_workspace_blobs_v1( + owner_id.into(), + workspace_id.into(), + 1000, + Some(first_page.last().unwrap().key.clone()), + ) + .await + .unwrap(); + assert_eq!(first_page.len(), 1000); + assert_eq!(last_page.len(), 1); + assert_eq!(last_page[0].key, "managed-key"); + let keys = first_page + .iter() + .chain(last_page.iter()) + .map(|blob| &blob.key) + .collect::>(); + assert_eq!(keys.len(), 1001); sqlx::query("DELETE FROM blobs WHERE workspace_id=$1 AND key LIKE 'managed-cap-%'") .bind(workspace_id) .execute(&pool) @@ -1705,7 +1721,7 @@ async fn strict_blob_management_authorizes_inventory_and_denies_before_object_cl ); assert!( runtime - .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into()) + .list_managed_workspace_blobs_v1(owner_id.into(), workspace_id.into(), 1000, None) .await .unwrap() .is_empty() diff --git a/packages/backend/native/src/runtime/config/file.rs b/packages/backend/native/src/runtime/config/file.rs index 086710e695..d0e6eb3c6c 100644 --- a/packages/backend/native/src/runtime/config/file.rs +++ b/packages/backend/native/src/runtime/config/file.rs @@ -28,7 +28,8 @@ impl ServerConfig { let deployment = match baseline.pointer("/deployment/type").and_then(serde_json::Value::as_str) { Some("cloud") => Deployment::Cloud, Some("selfhosted") => Deployment::SelfHosted, - // TODO(0.27.5): Remove the old deployment env fallback after self-host config files are upgraded. + // TODO(0.27.5): Remove the old deployment env fallback after self-host + // config files are upgraded. None => match legacy_deployment_type { Some("cloud") => Deployment::Cloud, Some("selfhosted") => Deployment::SelfHosted, diff --git a/packages/backend/native/src/runtime/config/store.rs b/packages/backend/native/src/runtime/config/store.rs index ad8a782a0a..5ab94f9998 100644 --- a/packages/backend/native/src/runtime/config/store.rs +++ b/packages/backend/native/src/runtime/config/store.rs @@ -34,7 +34,8 @@ pub(super) const NATIVE_APP_CONFIG_KEYS: &[&str] = &[ "copilot.providers.profiles", "copilot.storage", "crypto.privateKey", - // TODO(0.27.5): Remove legacy DB override loading after old instances exit and existing rows are cleared. + // TODO(0.27.5): Remove legacy DB override loading after old instances exit + // and existing rows are cleared. "db.datasourceUrl", "indexer.enabled", "indexer.provider.apiKey", @@ -49,7 +50,8 @@ pub(super) const NATIVE_APP_CONFIG_KEYS: &[&str] = &[ "payment.enabled", "payment.revenuecat", "payment.stripe", - // TODO(0.27.5): Remove legacy Redis DB override loading after old instances exit and existing rows are cleared. + // TODO(0.27.5): Remove legacy Redis DB override loading after old instances + // exit and existing rows are cleared. "redis.db", "redis.host", "redis.ioredis", diff --git a/packages/backend/native/src/runtime/object_storage/types.rs b/packages/backend/native/src/runtime/object_storage/types.rs index 271d5f0fdf..4521c6dac3 100644 --- a/packages/backend/native/src/runtime/object_storage/types.rs +++ b/packages/backend/native/src/runtime/object_storage/types.rs @@ -231,7 +231,8 @@ fn validate_copilot_key(key: &str) -> bool { // chat attachments, generated images, transcript slices: // //[-] [user_id, workspace_id, hash] => is_id_segment(user_id) && is_id_segment(workspace_id) && is_hash_or_slice(hash), - // embedding workspace file: workspace-files/// + // embedding workspace file: + // workspace-files/// ["workspace-files", workspace_id, file_id, hash] => { is_id_segment(workspace_id) && is_id_segment(file_id) && is_sha256_base64url(hash) } diff --git a/packages/backend/native/src/runtime/types.rs b/packages/backend/native/src/runtime/types.rs index 7e7edea018..8df32ced89 100644 --- a/packages/backend/native/src/runtime/types.rs +++ b/packages/backend/native/src/runtime/types.rs @@ -72,6 +72,8 @@ pub struct DocumentEmbeddingProjectionInput { pub doc_id: String, pub revision: String, pub source_hash: String, + pub skip_reason: Option, + pub root_revision: Option, pub units: Vec, pub deleted: Option, } diff --git a/packages/backend/server/src/base/logger/__tests__/service.spec.ts b/packages/backend/server/src/base/logger/__tests__/service.spec.ts index 6d8c5489db..186e7cf1cf 100644 --- a/packages/backend/server/src/base/logger/__tests__/service.spec.ts +++ b/packages/backend/server/src/base/logger/__tests__/service.spec.ts @@ -1,9 +1,12 @@ +import { PassThrough } from 'node:stream'; import { mock } from 'node:test'; import { TestingModule } from '@nestjs/testing'; import ava, { TestFn } from 'ava'; +import { createLogger, format, transports } from 'winston'; import { createTestingModule } from '../../../__tests__/utils'; +import { AFFiNELogger as CloudLogger } from '../../../plugins/gcloud/logging/logger'; import { AFFiNELogger } from '../service'; export const test = ava as TestFn<{ @@ -73,3 +76,39 @@ test('should print error stack with cause', t => { `${error.stack}\n\nCaused by:\n\n${(error.cause as any).stack}` ); }); + +test('cloud logger emits structured context and error causes in one record', t => { + const output = new PassThrough(); + let json = ''; + output.on('data', chunk => { + json += chunk.toString(); + }); + const raw = createLogger({ + format: format.json(), + transports: [new transports.Stream({ stream: output })], + }); + const logger = new CloudLogger(raw); + const cause = new Error('source missing'); + const error = new Error('write failed', { cause }); + logger.error( + 'Failed to insert doc updates', + { + workspaceId: 'workspace', + docId: 'doc', + writeIntent: 'create_doc', + error, + }, + 'DocStorage' + ); + const record = JSON.parse(json); + t.like(record, { + message: 'Failed to insert doc updates', + workspaceId: 'workspace', + docId: 'doc', + writeIntent: 'create_doc', + context: 'DocStorage', + }); + t.is(record.stack[0], `${error.stack}\n\nCaused by:\n\n${cause.stack}`); + raw.close(); + output.destroy(); +}); diff --git a/packages/backend/server/src/base/logger/service.ts b/packages/backend/server/src/base/logger/service.ts index b0b6962cc0..fedf7acb46 100644 --- a/packages/backend/server/src/base/logger/service.ts +++ b/packages/backend/server/src/base/logger/service.ts @@ -39,20 +39,19 @@ export class AFFiNELogger extends ConsoleLogger { return stackOrError; } - /** - * Nestjs ConsoleLogger.error() will not print the stack trace if the error is an instance of Error - * This method is a workaround to print the stack trace - * - * Usage: - * ``` - * this.logger.error('some error happens', errInstance); - * ``` - */ - override error( - message: any, - stackOrError?: Error | string | unknown, - context?: string - ) { - super.error(message, AFFiNELogger.formatStack(stackOrError), context); + static formatError(message: unknown, details: unknown) { + if (details && typeof details === 'object' && !(details instanceof Error)) { + const { error, ...fields } = details as Record; + return { + message: { ...fields, message }, + stack: AFFiNELogger.formatStack(error), + }; + } + return { message, stack: AFFiNELogger.formatStack(details) }; + } + + override error(message: unknown, details?: unknown, context?: string) { + const formatted = AFFiNELogger.formatError(message, details); + super.error(formatted.message, formatted.stack, context); } } diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts index 391f705a66..97e7f20971 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/job.spec.ts @@ -4,6 +4,15 @@ import { join } from 'node:path'; import { ScheduleModule } from '@nestjs/schedule'; import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; +import { + applyUpdate, + Array as YArray, + Doc, + encodeStateAsUpdate, + encodeStateVector, + Map as YMap, + Text as YText, +} from 'yjs'; import { createTestingModule, @@ -94,6 +103,15 @@ test.before(async t => { createdByUser: null, updatedByUser: null, }); + const root = new Doc(); + const pages = new YArray>(); + root.getMap('meta').set('pages', pages); + pages.push([new YMap([['id', 'doc-1']])]); + t.context.getSnapshot.withArgs('workspace-1', 'workspace-1').resolves({ + blob: Buffer.from(encodeStateAsUpdate(root)), + updatedAt: new Date('2026-01-02T00:00:00Z'), + }); + root.destroy(); t.context.allowEmbedding = Sinon.stub( models.workspace, 'allowEmbedding' @@ -126,12 +144,12 @@ test.after.always(async t => { test('backend-runtime jobs ingest documents and clean runtime state', async t => { await t.context.embeddingService.syncDocument('workspace-1', 'doc-1'); - t.is(t.context.getSnapshot.callCount, 1); + t.is(t.context.getSnapshot.callCount, 2); t.is(t.context.runtime.syncEmbeddingState.callCount, 1); t.like(t.context.runtime.syncEmbeddingState.firstCall.args[0], { workspaceId: 'workspace-1', enabled: true, - reconcileDocuments: true, + reconcileDocuments: false, }); t.is( t.context.runtime.syncEmbeddingState.firstCall.args[0].documents[0].docId, @@ -191,6 +209,88 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => ), { instanceOf: CopilotSelectedSourcesLimitExceeded } ); + const healthyRoot = await t.context.getSnapshot('workspace-1', 'workspace-1'); + const partialRoot = new Doc(); + partialRoot.getMap('meta').set('version', 1); + const priorState = encodeStateVector(partialRoot); + const rootPages = new YArray>(); + partialRoot.getMap('meta').set('pages', rootPages); + rootPages.push([ + new YMap([ + ['id', 'doc-1'], + ['trash', true], + ]), + ]); + t.context.getSnapshot.withArgs('workspace-1', 'workspace-1').resolves({ + ...healthyRoot, + blob: Buffer.from(encodeStateAsUpdate(partialRoot, priorState)), + }); + t.false( + (await t.context.embeddingService.inspectWorkspaceRoot('workspace-1')) + .complete + ); + t.context.getSnapshot.withArgs('workspace-1', 'workspace-1').resolves({ + ...healthyRoot, + blob: Buffer.from(encodeStateAsUpdate(partialRoot)), + }); + const repairedRoot = + await t.context.embeddingService.inspectWorkspaceRoot('workspace-1'); + t.true(repairedRoot.complete); + t.true(repairedRoot.docIds.has('doc-1')); + partialRoot.destroy(); + t.context.getSnapshot + .withArgs('workspace-1', 'workspace-1') + .resolves(healthyRoot); + const healthySnapshot = await t.context.getSnapshot('workspace-1', 'doc-1'); + const viewOnly = new Doc(); + viewOnly.getMap('view').set('mode', 'page'); + const nulDoc = new Doc(); + applyUpdate(nulDoc, healthySnapshot.blob); + const text = [...nulDoc.getMap>('blocks').values()] + .map(block => block.get('prop:text')) + .find((value): value is YText => value instanceof YText)!; + text.insert(0, '\0'); + for (const [blob, reason] of [ + [Buffer.from([0, 0]), 'empty_document'], + [Buffer.from([255]), 'invalid_document'], + [Buffer.from(encodeStateAsUpdate(viewOnly)), 'document_blocks_missing'], + [Buffer.from(encodeStateAsUpdate(nulDoc)), 'document_contains_nul'], + ] as const) { + t.context.getSnapshot + .withArgs('workspace-1', 'doc-1') + .resolves({ ...healthySnapshot, blob }); + await t.context.embeddingService.syncDocument('workspace-1', 'doc-1'); + t.like(t.context.runtime.syncEmbeddingState.lastCall.args[0].documents[0], { + docId: 'doc-1', + revision: healthySnapshot.updatedAt.getTime().toString(), + skipReason: reason, + units: [], + }); + } + viewOnly.destroy(); + nulDoc.destroy(); + t.context.getSnapshot + .withArgs('workspace-1', 'doc-1') + .resolves(healthySnapshot); + await t.context.embeddingService.syncDocument('workspace-1', 'doc-1', { + complete: false, + revision: 'root-1', + docIds: new Set(), + }); + t.like(t.context.runtime.syncEmbeddingState.lastCall.args[0].documents[0], { + skipReason: 'workspace_root_incomplete', + rootRevision: 'root-1', + units: [], + }); + await t.context.embeddingService.syncDocument('workspace-1', 'doc-1', { + complete: true, + revision: 'root-2', + docIds: new Set(['doc-1']), + }); + t.true( + t.context.runtime.syncEmbeddingState.lastCall.args[0].documents[0].units + .length > 0 + ); t.context.getSnapshot.resolves(null); await t.throwsAsync( () => @@ -204,11 +304,7 @@ test('backend-runtime jobs ingest documents and clean runtime state', async t => await t.context.embeddingService.syncDocument('workspace-1', 'missing-doc'); t.is( t.context.runtime.syncEmbeddingState.callCount, - callsBeforeMissingBackgroundDoc + 1 - ); - t.deepEqual( - t.context.runtime.syncEmbeddingState.lastCall.args[0].documents, - [] + callsBeforeMissingBackgroundDoc ); t.context.runtime.cleanupExpiredRuntimeStates.onCall(0).resolves(1000); diff --git a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts index e6be4637df..c8ec0118bc 100644 --- a/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts +++ b/packages/backend/server/src/core/backend-runtime/__tests__/provider.spec.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import test from 'ava'; import Sinon from 'sinon'; -import type { EventBus } from '../../../base'; +import { AccessDenied, type EventBus, NotFound } from '../../../base'; import { ServerConfigHandle } from '../../../native'; import { BackendRuntimeProvider } from '../provider'; @@ -113,6 +113,36 @@ test('backend-runtime provider measures explicit typed methods', async t => { t.true(runtime.assertCopilotRoute.calledOnceWithExactly(routeInput)); }); +test('blob authorization and missing sources retain HTTP error statuses', async t => { + const provider = new BackendRuntimeProvider(serverConfig); + const runtime = { + getDocBlobManifestV1: Sinon.stub(), + getBlobV1: Sinon.stub(), + }; + (provider as unknown as { runtime: typeof runtime }).runtime = runtime; + const source = { + type: 'currentDoc' as const, + workspaceId: 'workspace', + docId: 'doc', + }; + for (const [code, errorClass, status] of [ + ['blob_source_access_denied', AccessDenied, 403], + ['blob_source_not_found', NotFound, 404], + ] as const) { + runtime.getDocBlobManifestV1.rejects( + new Error(`[affine-runtime:${code}] ${code}`) + ); + runtime.getBlobV1.rejects(new Error(`[affine-runtime:${code}] ${code}`)); + for (const request of [ + () => provider.getDocBlobManifestV1(undefined, source), + () => provider.openBlobV1(undefined, source, 'blob'), + ]) { + const error = await t.throwsAsync(request, { instanceOf: errorClass }); + t.is(error?.status, status); + } + } +}); + test('backend-runtime provider encodes recursive search contracts at the native boundary', async t => { const provider = new BackendRuntimeProvider(serverConfig); const runtime = { diff --git a/packages/backend/server/src/core/backend-runtime/contracts.ts b/packages/backend/server/src/core/backend-runtime/contracts.ts index a08d7b65e0..bb840d9698 100644 --- a/packages/backend/server/src/core/backend-runtime/contracts.ts +++ b/packages/backend/server/src/core/backend-runtime/contracts.ts @@ -211,7 +211,9 @@ export type RuntimeMailDeliveryQuotaDecision = { export type RuntimeQuotaMethods = RuntimeInstance & { listManagedWorkspaceBlobsV1( actorUserId: string, - workspaceId: string + workspaceId: string, + first: number, + after?: string ): Promise< Array<{ key: string; mime: string; size: number; createdAt: string }> >; diff --git a/packages/backend/server/src/core/backend-runtime/job.ts b/packages/backend/server/src/core/backend-runtime/job.ts index 75870bde7d..d7c9c0169d 100644 --- a/packages/backend/server/src/core/backend-runtime/job.ts +++ b/packages/backend/server/src/core/backend-runtime/job.ts @@ -9,6 +9,10 @@ import { metrics, } from '../../base'; import { Models } from '../../models'; +import { + type DocumentEmbeddingProjectionInput, + projectWorkspaceRootFromBinary, +} from '../../native'; import { projectDocSearch } from '../utils/blocksuite'; import { BackendRuntimeProvider } from './provider'; @@ -28,8 +32,44 @@ export class BackendRuntimeEmbeddingService { private readonly models: Models ) {} - async syncDocument(workspaceId: string, docId: string) { - await this.syncDocuments(workspaceId, [docId], true); + async inspectWorkspaceRoot(workspaceId: string) { + const snapshot = await this.models.doc.getSnapshot( + workspaceId, + workspaceId + ); + const revision = snapshot?.updatedAt.getTime().toString() ?? 'missing'; + if (!snapshot) + return { revision, complete: false, docIds: new Set() }; + try { + const projection = projectWorkspaceRootFromBinary( + Buffer.from(snapshot.blob) + ); + return { + revision, + complete: projection.complete, + docIds: new Set(projection.docIds), + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message !== 'invalid_binary') throw error; + return { revision, complete: false, docIds: new Set() }; + } + } + + async syncDocument( + workspaceId: string, + docId: string, + root?: Awaited< + ReturnType + > + ) { + await this.syncDocuments( + workspaceId, + [docId], + false, + undefined, + root ?? (await this.inspectWorkspaceRoot(workspaceId)) + ); } async prepareSelectedDocuments(workspaceId: string, docIds: string[]) { @@ -65,7 +105,10 @@ export class BackendRuntimeEmbeddingService { workspaceId: string, docIds: string[], reconcileDocuments: boolean, - scheduling?: { priority: number; waitForReadyMs: number } + scheduling?: { priority: number; waitForReadyMs: number }, + root?: Awaited< + ReturnType + > ) { if (!(await this.rt.embeddingHealth()).enabled) { if (scheduling) throw new CopilotSelectedSourcesUnavailable(); @@ -76,7 +119,7 @@ export class BackendRuntimeEmbeddingService { if (scheduling) throw new CopilotSelectedSourcesUnavailable(); return; } - const documents = []; + const documents: DocumentEmbeddingProjectionInput[] = []; let unitCount = 0; let textBytes = 0; for (const docId of docIds) { @@ -86,7 +129,64 @@ export class BackendRuntimeEmbeddingService { continue; } const revision = snapshot.updatedAt.getTime().toString(); - const projection = projectDocSearch(snapshot.blob, docId, revision); + const skipped: DocumentEmbeddingProjectionInput = { + docId, + revision, + sourceHash: revision, + units: [], + }; + if (root && (!root.complete || !root.docIds.has(docId))) { + documents.push({ + ...skipped, + skipReason: root.complete + ? 'document_not_in_root' + : 'workspace_root_incomplete', + rootRevision: root.revision, + }); + continue; + } + if ( + !snapshot.blob.length || + (snapshot.blob.length === 2 && + snapshot.blob[0] === 0 && + snapshot.blob[1] === 0) + ) { + if (scheduling) throw new CopilotSelectedSourcesUnavailable(); + documents.push({ ...skipped, skipReason: 'empty_document' }); + continue; + } + let projection; + try { + projection = projectDocSearch(snapshot.blob, docId, revision); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if ( + scheduling || + !['invalid_binary', 'parser_error: blocks map not found'].includes( + message + ) + ) + throw error; + documents.push({ + ...skipped, + skipReason: + message === 'invalid_binary' + ? 'invalid_document' + : 'document_blocks_missing', + }); + continue; + } + if ( + projection.units.some(unit => + Object.values(unit).some( + value => typeof value === 'string' && value.includes('\0') + ) + ) + ) { + if (scheduling) throw new CopilotSelectedSourcesFailed(); + documents.push({ ...skipped, skipReason: 'document_contains_nul' }); + continue; + } unitCount += projection.units.length; for (const unit of projection.units) { textBytes += Buffer.byteLength(unit.text); @@ -95,7 +195,12 @@ export class BackendRuntimeEmbeddingService { unitCount > SELECTED_DOCUMENT_UNIT_LIMIT || textBytes > SELECTED_DOCUMENT_TEXT_BYTE_LIMIT ) { - throw new CopilotSelectedSourcesLimitExceeded(); + if (scheduling) throw new CopilotSelectedSourcesLimitExceeded(); + documents.push({ + ...skipped, + skipReason: 'document_projection_too_large', + }); + continue; } documents.push({ docId, diff --git a/packages/backend/server/src/core/backend-runtime/operations.ts b/packages/backend/server/src/core/backend-runtime/operations.ts index c14c932617..f9ccf6b273 100644 --- a/packages/backend/server/src/core/backend-runtime/operations.ts +++ b/packages/backend/server/src/core/backend-runtime/operations.ts @@ -1,3 +1,4 @@ +import { AccessDenied, BadRequest, NotFound } from '../../base/error'; import { wrapCallMetric } from '../../base/metrics'; import type { RuntimeUserQuotaState, @@ -389,11 +390,18 @@ export class BackendRuntimeCoreOperations { ); } - async listManagedWorkspaceBlobsV1(actorUserId: string, workspaceId: string) { + async listManagedWorkspaceBlobsV1( + actorUserId: string, + workspaceId: string, + first: number, + after?: string + ) { return await this.measured('listManagedWorkspaceBlobsV1', rt => this.quotaRuntime(rt).listManagedWorkspaceBlobsV1( actorUserId, - workspaceId + workspaceId, + first, + after ) ); } @@ -585,6 +593,15 @@ export class BackendRuntimeCoreOperations { const message = error instanceof Error ? error.message : String(error); const match = runtimeErrorPattern.exec(message); if (!match) throw error; + switch (match[1]) { + case 'blob_source_access_denied': + throw new AccessDenied('Blob source access denied'); + case 'blob_source_not_found': + throw new NotFound('Blob source not found'); + case 'blob_source_invalid': + case 'blob_source_type_invalid': + throw new BadRequest('Invalid blob source'); + } throw new BackendRuntimeError( match[1], message.replace(runtimeErrorPattern, ''), diff --git a/packages/backend/server/src/core/doc-jobs/job.ts b/packages/backend/server/src/core/doc-jobs/job.ts index ac4ba42752..29b7ea7e42 100644 --- a/packages/backend/server/src/core/doc-jobs/job.ts +++ b/packages/backend/server/src/core/doc-jobs/job.ts @@ -66,6 +66,20 @@ export class DocJobRunner { ORDER BY workspace.last_check_embeddings, workspace.sid LIMIT ${WORKSPACE_BATCH_SIZE} `; + const rootsByWorkspace = new Map< + string, + Awaited< + ReturnType + > + >(); + const inspectRoot = async (workspaceId: string) => { + let root = rootsByWorkspace.get(workspaceId); + if (!root) { + root = await this.embedding.inspectWorkspaceRoot(workspaceId); + rootsByWorkspace.set(workspaceId, root); + } + return root; + }; let reconciledWorkspaces = 0; for (const workspace of roots) { const claimedAt = new Date(); @@ -78,6 +92,8 @@ export class DocJobRunner { }); if (!claimed.count) continue; try { + const root = await inspectRoot(workspace.id); + if (!root.complete) continue; await this.embedding.reconcileDocuments(workspace.id); reconciledWorkspaces++; } catch (error) { @@ -85,10 +101,10 @@ export class DocJobRunner { where: { id: workspace.id, lastCheckEmbeddings: claimedAt }, data: { lastCheckEmbeddings: new Date(0) }, }); - this.logger.error( - `embedding workspace reconciliation failed workspace=${workspace.id}`, - error - ); + this.logger.error('embedding workspace reconciliation failed', { + workspaceId: workspace.id, + error, + }); } } @@ -98,6 +114,7 @@ export class DocJobRunner { SELECT snapshot.workspace_id AS "workspaceId", snapshot.guid AS "docId" FROM snapshots snapshot JOIN workspaces workspace ON workspace.id = snapshot.workspace_id + LEFT JOIN snapshots root ON root.workspace_id = snapshot.workspace_id AND root.guid = snapshot.workspace_id LEFT JOIN embedding_sources source ON source.workspace_id = snapshot.workspace_id AND source.source_kind = 'document' @@ -108,7 +125,8 @@ export class DocJobRunner { AND snapshot.guid NOT LIKE 'userdata$%' AND ( source.id IS NULL - OR source.deleted_at IS NOT NULL + OR (source.document_projection->>'rootRevision' IS NOT NULL + AND source.document_projection->>'rootRevision' <> COALESCE(((extract(epoch FROM root.updated_at) * 1000)::bigint)::text, 'missing')) OR source.content_revision <> ((extract(epoch FROM snapshot.updated_at) * 1000)::bigint)::text ) ORDER BY snapshot.updated_at, snapshot.workspace_id, snapshot.guid @@ -117,13 +135,17 @@ export class DocJobRunner { let syncedDocuments = 0; for (const document of documents) { try { - await this.embedding.syncDocument(document.workspaceId, document.docId); + await this.embedding.syncDocument( + document.workspaceId, + document.docId, + await inspectRoot(document.workspaceId) + ); syncedDocuments++; } catch (error) { - this.logger.error( - `document embedding sync failed workspace=${document.workspaceId} doc=${document.docId}`, - error - ); + this.logger.error('document embedding sync failed', { + ...document, + error, + }); } } return { @@ -168,10 +190,11 @@ export class DocJobRunner { }); } } catch (error) { - this.logger.error( - `document compaction failed workspace=${group.workspaceId} doc=${group.id}`, - error - ); + this.logger.error('document compaction failed', { + workspaceId: group.workspaceId, + docId: group.id, + error, + }); } } return { scanned: groups.length, merged }; @@ -211,10 +234,11 @@ export class DocJobRunner { await this.models.doc.upsertMeta(workspace.id, docId, content); fixed++; } catch (error) { - this.logger.error( - `document summary repair failed workspace=${workspace.id} doc=${docId}`, - error - ); + this.logger.error('document summary repair failed', { + workspaceId: workspace.id, + docId, + error, + }); } } } diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.ts index e12142bc38..863db79020 100644 --- a/packages/backend/server/src/core/doc/adapters/workspace.ts +++ b/packages/backend/server/src/core/doc/adapters/workspace.ts @@ -174,7 +174,20 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter { }); } } catch (e) { - this.logger.error('Failed to insert doc updates', e); + this.logger.error('Failed to insert doc updates', { + workspaceId, + docId, + actorUserId: editorId, + writeIntent: 'trusted' in contract ? 'trusted' : contract.writeIntent, + permissionDocId: + 'trusted' in contract ? docId : (contract.permissionDocId ?? docId), + expectedPermissionGeneration: + 'trusted' in contract + ? undefined + : contract.expectedPermissionGeneration, + updateCount: updates.length, + error: e, + }); metrics.doc.counter('doc_update_insert_failed').add(1); throw new FailedToSaveUpdates(); } diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.ts index 6d22cb8fe0..cf87923ec0 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts @@ -11,6 +11,7 @@ import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; import type { FileUpload } from '../../../base'; import { + BadRequest, BlobInvalid, BlobNotFound, BlobQuotaExceeded, @@ -53,9 +54,18 @@ export class WorkspaceBlobResolver { }) async blobs( @CurrentUser() user: CurrentUser, - @Parent() workspace: WorkspaceType + @Parent() workspace: WorkspaceType, + @Args('first', { type: () => Int, defaultValue: 1000 }) first: number, + @Args('after', { type: () => String, nullable: true }) after?: string ) { - return this.runtime.listManagedWorkspaceBlobsV1(user.id, workspace.id); + if (first < 1 || first > 1000) + throw new BadRequest('Blob page size must be between 1 and 1000'); + return this.runtime.listManagedWorkspaceBlobsV1( + user.id, + workspace.id, + first, + after + ); } @ResolveField(() => BlobUploadPart, { diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index fa0af4a3d5..135f990f1a 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -315,6 +315,9 @@ export const htmlSanitize = serverNativeModule.htmlSanitize; export const processImage = serverNativeModule.processImage; export const projectDocCanvasFromBinary = serverNativeModule.projectDocCanvasFromBinary; +export const projectWorkspaceRootFromBinary = + serverNativeModule.projectWorkspaceRootFromBinary; + export const projectDocSearchFromBinary = serverNativeModule.projectDocSearchFromBinary; export const parseYDocToMarkdown = serverNativeModule.parseDocToMarkdown; diff --git a/packages/backend/server/src/plugins/gcloud/logging/logger.ts b/packages/backend/server/src/plugins/gcloud/logging/logger.ts index 0b334edd8d..ffd248a11b 100644 --- a/packages/backend/server/src/plugins/gcloud/logging/logger.ts +++ b/packages/backend/server/src/plugins/gcloud/logging/logger.ts @@ -3,15 +3,8 @@ import { WinstonLogger } from 'nest-winston'; import { AFFiNELogger as RawAFFiNELogger } from '../../../base/logger'; export class AFFiNELogger extends WinstonLogger { - override error( - message: any, - stackOrError?: Error | string | unknown, - context?: string - ) { - super.error( - message, - RawAFFiNELogger.formatStack(stackOrError) as string, - context - ); + override error(message: unknown, details?: unknown, context?: string) { + const formatted = RawAFFiNELogger.formatError(message, details); + super.error(formatted.message, formatted.stack as string, context); } } diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index b5573e7272..2b32b77317 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -2920,7 +2920,7 @@ type WorkspaceType { blobUploadPartUrl(key: String!, partNumber: Int!, uploadId: String!): BlobUploadPart! """List blobs of workspace""" - blobs: [ListedBlob!]! + blobs(after: String, first: Int! = 1000): [ListedBlob!]! byokSettings: WorkspaceByokSettingsType! byokUsage(from: DateTime!, to: DateTime!): [WorkspaceByokUsagePointType!]! calendars: [WorkspaceCalendarObjectType!]! diff --git a/packages/common/graphql/src/graphql/blob-list.gql b/packages/common/graphql/src/graphql/blob-list.gql index 5b314f0cc5..9cb99c6ed2 100644 --- a/packages/common/graphql/src/graphql/blob-list.gql +++ b/packages/common/graphql/src/graphql/blob-list.gql @@ -1,6 +1,6 @@ -query listBlobs($workspaceId: String!) { +query listBlobs($workspaceId: String!, $first: Int!, $after: String) { workspace(id: $workspaceId) { - blobs { + blobs(first: $first, after: $after) { key size mime diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index 4403dd5ff2..6ea7ff6876 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -592,9 +592,9 @@ export const deleteBlobMutation = { export const listBlobsQuery = { id: 'listBlobsQuery' as const, op: 'listBlobs', - query: `query listBlobs($workspaceId: String!) { + query: `query listBlobs($workspaceId: String!, $first: Int!, $after: String) { workspace(id: $workspaceId) { - blobs { + blobs(first: $first, after: $after) { key size mime diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index 0740f089c9..2fda79b11a 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -3730,6 +3730,11 @@ export interface WorkspaceTypeBlobUploadPartUrlArgs { uploadId: Scalars['String']['input']; } +export interface WorkspaceTypeBlobsArgs { + after?: InputMaybe; + first?: Scalars['Int']['input']; +} + export interface WorkspaceTypeByokUsageArgs { from: Scalars['DateTime']['input']; to: Scalars['DateTime']['input']; @@ -4390,6 +4395,8 @@ export type DeleteBlobMutation = { export type ListBlobsQueryVariables = Exact<{ workspaceId: Scalars['String']['input']; + first: Scalars['Int']['input']; + after?: InputMaybe; }>; export type ListBlobsQuery = { diff --git a/packages/common/nbstore/src/__tests__/cloud-blob.spec.ts b/packages/common/nbstore/src/__tests__/cloud-blob.spec.ts index e43fc7b6da..26c7631371 100644 --- a/packages/common/nbstore/src/__tests__/cloud-blob.spec.ts +++ b/packages/common/nbstore/src/__tests__/cloud-blob.spec.ts @@ -725,38 +725,33 @@ test('pages readable workspace sources without workspace inventory fallback', as ); }); -test('uses the managed workspace inventory only through its typed query', async () => { +test('paginates the managed workspace inventory', async () => { const storage = createStorage(); - const gqlMock = vi.fn(async ({ query }) => { + const records = Array.from({ length: 1001 }, (_, index) => ({ + key: `managed-${index.toString().padStart(4, '0')}`, + mime: 'text/plain', + size: 7, + createdAt: '2026-08-29T00:00:00.000Z', + })); + const gqlMock = vi.fn(async ({ query, variables }) => { expect(query).toBe(listBlobsQuery); + const start = variables.after + ? records.findIndex(row => row.key === variables.after) + 1 + : 0; return { - workspace: { - blobs: [ - { - key: 'managed', - mime: 'text/plain', - size: 7, - createdAt: '2026-08-29T00:00:00.000Z', - }, - ], - }, + workspace: { blobs: records.slice(start, start + variables.first) }, }; }); vi.spyOn(storage.connection, 'gql').mockImplementation( gqlMock as typeof storage.connection.gql ); - - expect(await storage.listManageable()).toEqual([ - { - key: 'managed', - mime: 'text/plain', - size: 7, - createdAt: new Date('2026-08-29T00:00:00.000Z'), - }, - ]); - await expect(storage.list()).rejects.toThrow( - 'Workspace blob inventory is unavailable' + expect(await storage.listManageable()).toEqual( + records.map(row => ({ ...row, createdAt: new Date(row.createdAt) })) ); + expect(gqlMock.mock.calls.map(([{ variables }]) => variables)).toEqual([ + { workspaceId: 'workspace-1', first: 1000, after: undefined }, + { workspaceId: 'workspace-1', first: 1000, after: 'managed-0999' }, + ]); }); test('releases transient workspace sources and preserves owned registrations', async () => { diff --git a/packages/common/nbstore/src/impls/cloud/blob-writer.ts b/packages/common/nbstore/src/impls/cloud/blob-writer.ts index 0057ef16b3..a9110ab362 100644 --- a/packages/common/nbstore/src/impls/cloud/blob-writer.ts +++ b/packages/common/nbstore/src/impls/cloud/blob-writer.ts @@ -14,6 +14,7 @@ import { import { type BlobRecord, + type ListedBlobRecord, OverCapacityError, OverSizeError, } from '../../storage'; @@ -173,14 +174,24 @@ export class CloudBlobWriter { } async listManageable() { - const result = await this.connection.gql({ - query: listBlobsQuery, - variables: { workspaceId: this.workspaceId }, - }); - return result.workspace.blobs.map(blob => ({ - ...blob, - createdAt: blob.createdAt ? new Date(blob.createdAt) : undefined, - })); + const blobs: ListedBlobRecord[] = []; + const first = 1000; + let after: string | undefined; + while (true) { + const result = await this.connection.gql({ + query: listBlobsQuery, + variables: { workspaceId: this.workspaceId, first, after }, + }); + const page = result.workspace.blobs; + blobs.push( + ...page.map(blob => ({ + ...blob, + createdAt: blob.createdAt ? new Date(blob.createdAt) : undefined, + })) + ); + if (page.length < first) return blobs; + after = page[page.length - 1].key; + } } private async uploadViaGraphql(blob: BlobRecord, signal?: AbortSignal) { diff --git a/packages/frontend/apps/electron/forge.config.mjs b/packages/frontend/apps/electron/forge.config.mjs index e97eef7e55..50639ab3a6 100644 --- a/packages/frontend/apps/electron/forge.config.mjs +++ b/packages/frontend/apps/electron/forge.config.mjs @@ -328,10 +328,12 @@ export default { name: productName, appBundleId: fromBuildIdentifier(appIdMap), icon: icnsPath, - osxSign: { - identity: 'Developer ID Application: TOEVERYTHING PTE. LTD.', - 'hardened-runtime': true, - }, + osxSign: process.env.AFFINE_UNSIGNED_PACKAGE + ? undefined + : { + identity: 'Developer ID Application: TOEVERYTHING PTE. LTD.', + 'hardened-runtime': true, + }, electronZipDir: process.env.ELECTRON_FORGE_ELECTRON_ZIP_DIR, osxNotarize: process.env.APPLE_ID ? { diff --git a/packages/frontend/apps/electron/package.json b/packages/frontend/apps/electron/package.json index f059e2f770..bf783200a8 100644 --- a/packages/frontend/apps/electron/package.json +++ b/packages/frontend/apps/electron/package.json @@ -30,7 +30,9 @@ "package": "electron-forge package", "make": "electron-forge make", "make-squirrel": "r ./scripts/make-squirrel.ts", - "make-nsis": "r ./scripts/make-nsis.ts" + "make-nsis": "r ./scripts/make-nsis.ts", + "sign-macos": "r ./scripts/sign-macos.ts", + "sign-windows": "r ./scripts/sign-windows.ts" }, "devDependencies": { "@affine-tools/cli": "workspace:*", @@ -49,6 +51,8 @@ "@electron-forge/plugin-auto-unpack-natives": "^7.11.1", "@electron-forge/plugin-fuses": "^7.11.1", "@electron-forge/shared-types": "^7.11.1", + "@electron/notarize": "2.5.0", + "@electron/osx-sign": "1.3.3", "@reforged/maker-appimage": "^5.2.0", "@sentry/electron": "^7.20.1", "@sentry/esbuild-plugin": "^5.3.0", diff --git a/packages/frontend/apps/electron/scripts/sign-macos.ts b/packages/frontend/apps/electron/scripts/sign-macos.ts new file mode 100644 index 0000000000..3fc28be749 --- /dev/null +++ b/packages/frontend/apps/electron/scripts/sign-macos.ts @@ -0,0 +1,33 @@ +import path from 'node:path'; + +import { notarize } from '@electron/notarize'; +import { signAsync } from '@electron/osx-sign'; + +import { arch, buildType, productName, ROOT } from './make-env.js'; + +const appPath = path.join( + ROOT, + 'out', + buildType, + `${productName}-darwin-${arch}`, + `${productName}.app` +); + +const { APPLE_ID, APPLE_PASSWORD, APPLE_TEAM_ID } = process.env; +if (!APPLE_ID || !APPLE_PASSWORD || !APPLE_TEAM_ID) { + throw new Error( + 'APPLE_ID, APPLE_PASSWORD and APPLE_TEAM_ID are required for notarization' + ); +} + +await signAsync({ + app: appPath, + identity: 'Developer ID Application: TOEVERYTHING PTE. LTD.', + optionsForFile: () => ({ hardenedRuntime: true }), +}); +await notarize({ + appPath, + appleId: APPLE_ID, + appleIdPassword: APPLE_PASSWORD, + teamId: APPLE_TEAM_ID, +}); diff --git a/packages/frontend/apps/electron/scripts/sign-windows.ts b/packages/frontend/apps/electron/scripts/sign-windows.ts new file mode 100644 index 0000000000..ac414aca0d --- /dev/null +++ b/packages/frontend/apps/electron/scripts/sign-windows.ts @@ -0,0 +1,84 @@ +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { readdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { buildType, ROOT } from './make-env.js'; + +const { + AFFINE_SIGNER_ADDR: address, + AFFINE_SIGNER_TOKEN: token, + AFFINE_SIGN_CLIENT_HASH: clientHash, + WINDOWS_SIGNER_PUBLIC_CERT_BASE64: certificate, +} = process.env; +if (!address || !token || !clientHash || !certificate) { + throw new Error( + 'Windows signer address, token, client hash and certificate are required' + ); +} + +const client = path.join(ROOT, 'affine-sign-client.exe'); +const cert = path.join(ROOT, 'windows-signer-public.cer'); +if (!existsSync(client)) { + const response = await fetch( + `https://cdn.affine.pro/sign-client/${clientHash}` + ); + if (!response.ok) + throw new Error(`Signer download failed: ${response.status}`); + await writeFile(client, Buffer.from(await response.arrayBuffer())); +} +await writeFile(cert, Buffer.from(certificate, 'base64')); + +if (!process.env.SIGNTOOL) { + const kits = path.join( + process.env['ProgramFiles(x86)'] ?? 'C:\\Program Files (x86)', + 'Windows Kits', + '10', + 'bin' + ); + const versions = (await readdir(kits)).sort().reverse(); + const signtool = versions + .map(version => path.join(kits, version, 'x64', 'signtool.exe')) + .find(file => existsSync(file)); + if (!signtool) throw new Error(`signtool.exe was not found in ${kits}`); + process.env.SIGNTOOL = signtool; +} + +const workdir = path.join( + ROOT, + 'out', + ...(process.argv.includes('--installers') ? [buildType, 'make'] : []) +); +const files = (await readdir(workdir, { recursive: true })) + .filter(file => + ['.exe', '.node', '.dll', '.msi'].includes(path.extname(file)) + ) + .map(file => `"${file}"`) + .join(' '); +if (!files) throw new Error(`No Windows binaries found in ${workdir}`); +await new Promise((resolve, reject) => { + const signer = spawn( + client, + [ + '--server', + address, + '--token', + token, + '--workdir', + workdir, + '--files', + files, + '--cert', + cert, + '--plain-tcp', + ], + { stdio: 'inherit' } + ); + signer.once('error', () => + reject(new Error('Could not start Windows signer')) + ); + signer.once('exit', code => { + if (code === 0) resolve(); + else reject(new Error(`Windows signing failed with exit code ${code}`)); + }); +}); diff --git a/packages/frontend/apps/ios/App/fastlane/Fastfile b/packages/frontend/apps/ios/App/fastlane/Fastfile index 879200ebfd..4e06af14d7 100644 --- a/packages/frontend/apps/ios/App/fastlane/Fastfile +++ b/packages/frontend/apps/ios/App/fastlane/Fastfile @@ -1,37 +1,80 @@ -# This file contains the fastlane.tools configuration -# You can find the documentation at https://docs.fastlane.tools -# -# For a list of all available actions, check out -# -# https://docs.fastlane.tools/actions -# -# For a list of all available plugins, check out -# -# https://docs.fastlane.tools/plugins/available-plugins -# - -# Uncomment the line if you want fastlane to automatically update itself -# update_fastlane +def release_api_key + app_store_connect_api_key( + key_id: ENV.fetch('APPLE_STORE_CONNECT_API_KEY_ID'), + issuer_id: ENV.fetch('APPLE_STORE_CONNECT_API_ISSUER_ID'), + key_content: ENV.fetch('APPLE_STORE_CONNECT_API_KEY'), + is_key_content_base64: true, + duration: 1200, + in_house: false + ) +end default_platform(:ios) platform :ios do - desc "Push a new beta build to TestFlight" - lane :beta do - api_key = app_store_connect_api_key( - key_id: ENV['APPLE_STORE_CONNECT_API_KEY_ID'], - issuer_id: ENV['APPLE_STORE_CONNECT_API_ISSUER_ID'], - key_content: ENV['APPLE_STORE_CONNECT_API_KEY'], - is_key_content_base64: true, - duration: 1200, # optional (maximum 1200) - in_house: false # optional but may be required if using match/sigh - ) - current_build_number = app_store_build_number(live: false) - increment_build_number(xcodeproj: "App.xcodeproj", build_number: current_build_number + 1) + desc 'Build a signed candidate without submitting it to TestFlight' + lane :archive do + provisioning_profiles = {} + { + 'AFFiNE' => ['app.affine.pro', 'BUILD_PROVISION_PROFILE_PATH'], + 'ShareExtension' => ['app.affine.pro.ShareExtension', 'BUILD_PROVISION_PROFILE_SHARE_EXTENSION_PATH'] + }.each do |target, (bundle_identifier, profile_env)| + profile_path = ENV.fetch(profile_env) + profile = FastlaneCore::ProvisioningProfile.parse(profile_path) + install_provisioning_profile(path: profile_path) + update_code_signing_settings( + path: 'App.xcodeproj', + targets: [target], + build_configurations: ['Release'], + use_automatic_signing: false, + code_sign_identity: 'Apple Distribution', + team_id: profile.fetch('TeamIdentifier').first, + profile_name: profile.fetch('UUID') + ) + provisioning_profiles[bundle_identifier] = profile.fetch('UUID') + end + api_key = release_api_key + current_build_number = app_store_build_number(live: false, api_key: api_key) + increment_build_number(xcodeproj: 'App.xcodeproj', build_number: current_build_number + 1) build_app( - workspace: "App.xcworkspace", - scheme: "App" + workspace: 'App.xcworkspace', scheme: 'App', configuration: 'Release', + output_directory: 'release', output_name: 'AFFiNE.ipa', + export_method: 'app-store', skip_profile_detection: true, + export_options: {signingStyle: 'manual', provisioningProfiles: provisioning_profiles} ) - upload_to_testflight(skip_waiting_for_build_processing: true, api_key: api_key) if ENV['BUILD_TARGET'] == 'distribution' + File.write('release/store-build.json', JSON.generate({ + appIdentifier: CredentialsManager::AppfileConfig.try_fetch_value(:app_identifier), + version: get_version_number(xcodeproj: 'App.xcodeproj', target: 'AFFiNE'), + buildNumber: (current_build_number + 1).to_s + })) + end + + desc 'Submit the prepared candidate and reconcile its exact App Store build' + lane :submit do + operation = { + operationRunId: Integer(ENV.fetch('STORE_OPERATION_RUN_ID')), + operationAttempt: Integer(ENV.fetch('STORE_OPERATION_ATTEMPT')) + } + UI.user_error!('A positive store operation run ID and attempt are required') unless operation.values.all?(&:positive?) + api_key = release_api_key + candidate = JSON.parse(File.read(ENV.fetch('STORE_BUILD_METADATA'))) + app = Spaceship::ConnectAPI::App.find(candidate.fetch('appIdentifier')) + UI.user_error!('App Store application not found') unless app + builds = Spaceship::ConnectAPI::Build.all( + app_id: app.id, version: candidate.fetch('version'), build_number: candidate.fetch('buildNumber') + ) + if ENV['STORE_QUERY_ONLY'] == '1' + UI.user_error!('No visible store result; the previous submission step ran, so its result remains unresolved') if builds.empty? && ENV['STORE_ALLOW_ABSENT'] != '1' + File.write(ENV.fetch('STORE_RECEIPT'), JSON.generate(candidate.merge({ + status: builds.empty? ? 'not-submitted' : 'submitted', + store: 'app-store-connect', **operation + }))) + next + end + if builds.empty? + upload_to_testflight(ipa: ENV.fetch('STORE_IPA'), skip_waiting_for_build_processing: true, api_key: api_key) + end + UI.user_error!('App Store build was rejected') if builds.any? { |build| ['FAILED', 'INVALID'].include?(build.processing_state) } + File.write(ENV.fetch('STORE_RECEIPT'), JSON.generate(candidate.merge({status: 'submitted', store: 'app-store-connect', **operation}))) end end diff --git a/packages/frontend/apps/ios/update_code_sign_identity.sh b/packages/frontend/apps/ios/update_code_sign_identity.sh deleted file mode 100755 index 9bd70c0213..0000000000 --- a/packages/frontend/apps/ios/update_code_sign_identity.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Script to update code signing settings in iOS project for CI environment -# Updates project.pbxproj to use Apple Distribution certificate and team settings - -set -e # Exit immediately if a command exits with a non-zero status - -# Define target values -TARGET_IDENTITY="Apple Distribution: TOEVERYTHING PTE. LTD. (73YMMDVT2M)" -TARGET_SIGN_STYLE="Manual" -TARGET_TEAM="73YMMDVT2M" -TARGET_PROVISIONING_PROFILE="AppStore app.affine.pro" - -# Get script directory and build absolute path to project file -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -PBXPROJ_FILE="$SCRIPT_DIR/App/App.xcodeproj/project.pbxproj" - -# Check if file exists -if [ ! -f "$PBXPROJ_FILE" ]; then - echo "❌ Error: project.pbxproj file not found: $PBXPROJ_FILE" - exit 1 -fi - -echo "🔍 Found project file: $PBXPROJ_FILE" - -# Display current settings -echo "📋 Current code signing settings:" -echo "--- CODE_SIGN_IDENTITY ---" -grep -n "CODE_SIGN_IDENTITY" "$PBXPROJ_FILE" || echo "No CODE_SIGN_IDENTITY settings found" -echo "--- CODE_SIGN_STYLE ---" -grep -n "CODE_SIGN_STYLE" "$PBXPROJ_FILE" || echo "No CODE_SIGN_STYLE settings found" -echo "--- DEVELOPMENT_TEAM ---" -grep -n "DEVELOPMENT_TEAM" "$PBXPROJ_FILE" || echo "No DEVELOPMENT_TEAM settings found" -echo "--- PROVISIONING_PROFILE_SPECIFIER ---" -grep -n "PROVISIONING_PROFILE_SPECIFIER" "$PBXPROJ_FILE" || echo "No PROVISIONING_PROFILE_SPECIFIER settings found" - -# Replace CODE_SIGN_IDENTITY settings -echo "" -echo "🔄 Replacing CODE_SIGN_IDENTITY..." -sed -i.tmp 's/CODE_SIGN_IDENTITY = "[^"]*";/CODE_SIGN_IDENTITY = "'"$TARGET_IDENTITY"'";/g' "$PBXPROJ_FILE" - -# Replace CODE_SIGN_STYLE settings -echo "🔄 Replacing CODE_SIGN_STYLE..." -sed -i.tmp 's/CODE_SIGN_STYLE = [^;]*;/CODE_SIGN_STYLE = '"$TARGET_SIGN_STYLE"';/g' "$PBXPROJ_FILE" - -# Replace DEVELOPMENT_TEAM settings -echo "🔄 Replacing DEVELOPMENT_TEAM..." -sed -i.tmp 's/DEVELOPMENT_TEAM = [^;]*;/DEVELOPMENT_TEAM = '"$TARGET_TEAM"';/g' "$PBXPROJ_FILE" - -# Replace PROVISIONING_PROFILE_SPECIFIER settings -echo "🔄 Replacing PROVISIONING_PROFILE_SPECIFIER..." -sed -i.tmp 's/PROVISIONING_PROFILE_SPECIFIER = "[^"]*";/PROVISIONING_PROFILE_SPECIFIER = "'"$TARGET_PROVISIONING_PROFILE"'";/g' "$PBXPROJ_FILE" - -# Remove temporary file -rm -f "${PBXPROJ_FILE}.tmp" - -# Verify replacement results -echo "" -echo "✅ Replacement completed! New code signing settings:" -echo "--- CODE_SIGN_IDENTITY ---" -grep -n "CODE_SIGN_IDENTITY" "$PBXPROJ_FILE" -echo "--- CODE_SIGN_STYLE ---" -grep -n "CODE_SIGN_STYLE" "$PBXPROJ_FILE" -echo "--- DEVELOPMENT_TEAM ---" -grep -n "DEVELOPMENT_TEAM" "$PBXPROJ_FILE" -echo "--- PROVISIONING_PROFILE_SPECIFIER ---" -grep -n "PROVISIONING_PROFILE_SPECIFIER" "$PBXPROJ_FILE" - -# Count replacements -IDENTITY_COUNT=$(grep -c "CODE_SIGN_IDENTITY.*$TARGET_IDENTITY" "$PBXPROJ_FILE") -STYLE_COUNT=$(grep -c "CODE_SIGN_STYLE.*$TARGET_SIGN_STYLE" "$PBXPROJ_FILE") -TEAM_COUNT=$(grep -c "DEVELOPMENT_TEAM.*$TARGET_TEAM" "$PBXPROJ_FILE") -PROVISIONING_COUNT=$(grep -c "PROVISIONING_PROFILE_SPECIFIER.*$TARGET_PROVISIONING_PROFILE" "$PBXPROJ_FILE") - -echo "" -echo "📊 Replacement summary:" -echo " - CODE_SIGN_IDENTITY: $IDENTITY_COUNT replacements" -echo " - CODE_SIGN_STYLE: $STYLE_COUNT replacements" -echo " - DEVELOPMENT_TEAM: $TEAM_COUNT replacements" -echo " - PROVISIONING_PROFILE_SPECIFIER: $PROVISIONING_COUNT replacements" - -echo "" -echo "🎉 Script execution completed successfully!" diff --git a/packages/frontend/mobile-native/src/ffi_types.rs b/packages/frontend/mobile-native/src/ffi_types.rs index a29c96b585..03d19390bf 100644 --- a/packages/frontend/mobile-native/src/ffi_types.rs +++ b/packages/frontend/mobile-native/src/ffi_types.rs @@ -83,8 +83,8 @@ impl TryFrom for NbDocClock { #[derive(uniffi::Record)] pub struct Blob { pub key: String, - // base64 encoded data; on mobile large blobs this is a file-path token prefixed - // with "__AFFINE_BLOB_FILE__:" + // base64 encoded data; on mobile large blobs this is a file-path token + // prefixed with "__AFFINE_BLOB_FILE__:" pub data: String, pub mime: String, pub size: i64, diff --git a/scripts/generate-release-yml.mjs b/scripts/generate-release-yml.mjs index 791a975c98..a13d9e1868 100644 --- a/scripts/generate-release-yml.mjs +++ b/scripts/generate-release-yml.mjs @@ -11,6 +11,7 @@ const filenamesMapping = { }; const releaseFiles = ['zip', 'exe', 'dmg', 'appimage', 'deb', 'flatpak']; +const releaseMetadata = {}; const generateYml = platform => { const yml = { @@ -70,8 +71,14 @@ const generateYml = platform => { `releaseDate: ${yml.releaseDate}\n`; fs.writeFileSync(path.join(releaseDir, outputFileName), ymlStr); + releaseMetadata[outputFileName] = yml; }; generateYml('macos'); generateYml('linux'); generateYml('all'); + +fs.writeFileSync( + path.join(releaseDir, 'release-metadata.json'), + JSON.stringify(releaseMetadata) +); diff --git a/scripts/set-version.sh b/scripts/set-version.sh index 6c2e1dfc8c..434f5b7113 100755 --- a/scripts/set-version.sh +++ b/scripts/set-version.sh @@ -11,30 +11,6 @@ for DIR_ITEM in $(yarn workspaces list --json | jq -r '.location'); do fi done -update_app_version_in_helm_charts() { - local file_path=$1 - local new_version=$2 - - # Check if file exists - if [ ! -f "$file_path" ]; then - echo "Error: File does not exist at $file_path." - return 1 - fi - - echo "Updating $file_path with appVersion $new_version" - - # Use sed to replace the appVersion value with the new version. - sed -i.bak -E "s/^appVersion:[[:space:]]+[\"']?.*[\"']?$/appVersion: \"$new_version\"/" "$file_path" || { - # If the sed command fails, print an error message and exit with a non-zero status - echo "Error: Failed to update the appVersion in $file_path." - return 1 - } - - echo "appVersion in $file_path updated to $new_version" - - rm "$file_path".bak -} - update_app_stream_version() { local file_path=$1 local new_version=$2 @@ -144,9 +120,6 @@ fi echo "iOS MARKETING_VERSION: $ios_new_version (app version: $new_version)" -update_app_version_in_helm_charts ".github/helm/affine/Chart.yaml" "$new_version" -update_app_version_in_helm_charts ".github/helm/affine/charts/graphql/Chart.yaml" "$new_version" -update_app_version_in_helm_charts ".github/helm/affine/charts/front/Chart.yaml" "$new_version" update_app_stream_version "packages/frontend/apps/electron/resources/affine.metainfo.xml" "$new_version" diff --git a/tools/cli/package.json b/tools/cli/package.json index 7f3bc65fc5..79c55bcf32 100644 --- a/tools/cli/package.json +++ b/tools/cli/package.json @@ -16,7 +16,6 @@ }, "dependencies": { "@affine-tools/utils": "workspace:*", - "@affine/s3-compat": "workspace:*", "@napi-rs/simple-git": "^1.0.0", "@rspack/core": "^2.0.8", "@rspack/dev-server": "^2.1.0", @@ -31,7 +30,6 @@ "inquirer": "^12.3.0", "jsonc-parser": "^3.3.1", "lodash-es": "^4.17.23", - "mime-types": "^3.0.0", "oxfmt": "0.63.0", "postcss": "^8.5.18", "postcss-loader": "^8.2.1", @@ -46,7 +44,6 @@ }, "devDependencies": { "@types/lodash-es": "^4.17.12", - "@types/mime-types": "^3.0.0", "@types/node": "^22.0.0" } } diff --git a/tools/cli/src/bundle.ts b/tools/cli/src/bundle.ts index 005ba6ab0a..ab2d418a21 100644 --- a/tools/cli/src/bundle.ts +++ b/tools/cli/src/bundle.ts @@ -20,10 +20,6 @@ import { createNodeTargetConfig as createRspackNodeTargetConfig, createWorkerTargetConfig as createRspackWorkerTargetConfig, } from './rspack'; -import { - shouldUploadReleaseAssets, - uploadDistAssetsToS3, -} from './rspack-shared/s3-plugin.js'; type WorkerConfig = { name: string }; type CreateWorkerTargetConfig = (pkg: Package, entry: string) => WorkerConfig; @@ -35,21 +31,6 @@ function assertRspackSupportedPackage(pkg: Package) { assertRspackSupportedPackageName(pkg.name); } -function shouldUploadAssetsForPackage(pkg: Package): boolean { - return ( - !!process.env.R2_SECRET_ACCESS_KEY && shouldUploadReleaseAssets(pkg.name) - ); -} - -async function uploadAssetsForPackage(pkg: Package, logger: Logger) { - if (!shouldUploadAssetsForPackage(pkg)) { - return; - } - logger.info('Uploading dist assets to R2...'); - await uploadDistAssetsToS3(pkg.distPath.value); - logger.info('Uploaded dist assets to R2.'); -} - function getBaseWorkerConfigs( pkg: Package, createWorkerTargetConfig: CreateWorkerTargetConfig, @@ -268,7 +249,6 @@ export class BundleCommand extends PackageCommand { return; } console.log(stats.toString('minimal')); - await uploadAssetsForPackage(pkg, logger); } catch (error) { console.error(error); process.exit(1); diff --git a/tools/cli/src/rspack-shared/html-plugin.ts b/tools/cli/src/rspack-shared/html-plugin.ts index 6571a24e31..a77c6dbd82 100644 --- a/tools/cli/src/rspack-shared/html-plugin.ts +++ b/tools/cli/src/rspack-shared/html-plugin.ts @@ -51,14 +51,13 @@ export const getPublicPath = (BUILD_CONFIG: BUILD_CONFIG_TYPE) => { return '/'; } - switch (BUILD_TYPE) { - case 'stable': - return 'https://prod.affineassets.com/'; - case 'beta': - return 'https://beta.affineassets.com/'; - default: - return 'https://dev.affineassets.com/'; - } + const domain = + BUILD_TYPE === 'stable' + ? 'prod.affineassets.com' + : BUILD_TYPE === 'beta' + ? 'beta.affineassets.com' + : 'dev.affineassets.com'; + return `https://${domain}/${process.env.RELEASE_ASSET_PREFIX ?? ''}`; }; const DESCRIPTION = `There can be more than Notion and Miro. AFFiNE is a next-gen knowledge base that brings planning, sorting and creating all together.`; diff --git a/tools/cli/src/rspack-shared/s3-plugin.ts b/tools/cli/src/rspack-shared/s3-plugin.ts deleted file mode 100644 index 2c562a6342..0000000000 --- a/tools/cli/src/rspack-shared/s3-plugin.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { readdir, readFile } from 'node:fs/promises'; -import { join, relative, sep } from 'node:path'; - -import { createS3CompatClient } from '@affine/s3-compat'; -import { lookup } from 'mime-types'; - -export const R2_BUCKET = - process.env.R2_BUCKET ?? - (process.env.BUILD_TYPE === 'canary' ? 'assets-dev' : 'assets-prod'); - -const S3_UPLOAD_PACKAGE_NAMES = new Set([ - '@affine/web', - '@affine/mobile', - '@affine/admin', -]); -const MAX_UPLOAD_RETRIES = 3; -const UPLOAD_RETRY_BASE_DELAY_MS = 500; - -function createR2Client() { - const { R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = process.env; - if (!R2_ACCOUNT_ID || !R2_ACCESS_KEY_ID || !R2_SECRET_ACCESS_KEY) { - throw new Error('Missing R2 credentials for uploading release assets'); - } - - return createS3CompatClient( - { - region: 'auto', - bucket: R2_BUCKET, - forcePathStyle: true, - endpoint: `https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, - }, - { - accessKeyId: R2_ACCESS_KEY_ID, - secretAccessKey: R2_SECRET_ACCESS_KEY, - } - ); -} - -async function collectFiles(dir: string): Promise { - const dirs = [dir]; - const files: string[] = []; - - while (dirs.length > 0) { - const current = dirs.pop()!; - const entries = await readdir(current, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = join(current, entry.name); - if (entry.isDirectory()) { - dirs.push(fullPath); - } else if (entry.isFile()) { - files.push(fullPath); - } - } - } - - return files; -} - -function toAssetKey(outputPath: string, filePath: string): string { - return relative(outputPath, filePath).split(sep).join('/'); -} - -function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -async function putObjectWithRetry( - s3: ReturnType, - asset: string, - assetSource: Buffer, - contentType: string | false | undefined -) { - let retries = 0; - while (true) { - try { - await s3.putObject(asset, assetSource, { - contentType: contentType || undefined, - contentLength: assetSource.byteLength, - }); - return; - } catch (error) { - if (retries >= MAX_UPLOAD_RETRIES) { - throw error; - } - retries += 1; - const delay = UPLOAD_RETRY_BASE_DELAY_MS * 2 ** (retries - 1); - const errorMessage = - error instanceof Error ? error.message : String(error); - console.warn( - `[s3-upload] Retry ${retries}/${MAX_UPLOAD_RETRIES} for ${asset}: ${errorMessage}` - ); - await sleep(delay); - } - } -} - -async function runInParallel( - values: T[], - worker: (value: T) => Promise, - concurrency = 16 -) { - if (values.length === 0) { - return; - } - let nextIndex = 0; - const workers = Array.from( - { length: Math.min(concurrency, values.length) }, - async () => { - while (true) { - const index = nextIndex++; - if (index >= values.length) { - return; - } - await worker(values[index]!); - } - } - ); - - await Promise.all(workers); -} - -export function shouldUploadReleaseAssets(pkgName: string): boolean { - return S3_UPLOAD_PACKAGE_NAMES.has(pkgName); -} - -export async function uploadDistAssetsToS3(outputPath: string) { - const allFiles = await collectFiles(outputPath); - const uploadFiles = allFiles.filter(file => !file.endsWith('.html')); - - if (uploadFiles.length === 0) { - return; - } - - const s3 = createR2Client(); - await runInParallel(uploadFiles, async filePath => { - const asset = toAssetKey(outputPath, filePath); - const assetSource = await readFile(filePath); - const contentType = lookup(asset); - await putObjectWithRetry(s3, asset, assetSource, contentType); - }); -} diff --git a/tools/cli/tsconfig.json b/tools/cli/tsconfig.json index 006bdb5791..ab3bade10a 100644 --- a/tools/cli/tsconfig.json +++ b/tools/cli/tsconfig.json @@ -6,8 +6,5 @@ "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo" }, "include": ["./src"], - "references": [ - { "path": "../utils" }, - { "path": "../../packages/common/s3-compat" } - ] + "references": [{ "path": "../utils" }] } diff --git a/tools/playstore-auto-bump/index.ts b/tools/playstore-auto-bump/index.ts index 03d60547fe..b28803b0e6 100644 --- a/tools/playstore-auto-bump/index.ts +++ b/tools/playstore-auto-bump/index.ts @@ -1,4 +1,3 @@ -import { execSync } from 'node:child_process'; import fs from 'node:fs'; import { Package } from '@affine-tools/utils/workspace'; @@ -7,70 +6,156 @@ import { auth as google_auth, } from '@googleapis/androidpublisher'; -export async function fetchVersionCode(applicationId: string): Promise { - const auth = new google_auth.GoogleAuth({ - scopes: ['https://www.googleapis.com/auth/androidpublisher'], - }); - const androidPublisher = new androidpublisher_v3.Androidpublisher({ - auth, - }); - const appEdit = await androidPublisher.edits.insert({ - packageName: applicationId, - requestBody: { - // 20min - expiryTimeSeconds: (Math.floor(Date.now() / 1000) + 12000).toString(), - }, - }); +const packageName = 'app.affine.pro'; +const auth = new google_auth.GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/androidpublisher'], +}); +const publisher = new androidpublisher_v3.Androidpublisher({ auth }); - if (!appEdit.data.id) { - throw new Error('Failed to create edit'); +async function createEdit(applicationId: string) { + const response = await publisher.edits.insert({ packageName: applicationId }); + if (!response.data.id) { + throw new Error('Google Play returned no edit ID'); } - - const lists = await androidPublisher.edits.bundles.list({ - packageName: applicationId, - editId: appEdit.data.id, - }); - - let versionCode: number = 1; - try { - versionCode = - lists.data.bundles?.[lists.data.bundles.length - 1].versionCode || 1; - } catch {} - - console.info(`Remote version code: ${versionCode}`); - - console.info(`Writing edit ID to ${process.env.GITHUB_OUTPUT}`); - - if (process.env.GITHUB_OUTPUT) { - execSync( - `echo "EDIT_ID=${appEdit.data.id}" >> ${process.env.GITHUB_OUTPUT}`, - { - stdio: 'inherit', - } - ); - } - - return versionCode; + return response.data.id; } -const versionCodeRegexPattern = /(versionCode(?:\s|=)*)(.*)/; +export async function fetchVersionCode(applicationId: string): Promise { + const editId = await createEdit(applicationId); + const response = await publisher.edits.bundles.list({ + packageName: applicationId, + editId, + }); + await publisher.edits.delete({ packageName: applicationId, editId }); + return Math.max( + 1, + ...(response.data.bundles ?? []).map(bundle => bundle.versionCode ?? 0) + ); +} -const gradlePath = new Package('@affine/android').join( - 'App/app/build.gradle' -).value; +async function bump() { + const gradlePath = new Package('@affine/android').join( + 'App/app/build.gradle' + ).value; + const gradle = fs.readFileSync(gradlePath, 'utf8'); + const pattern = /(versionCode(?:\s|=)*)(\d+)/; + const local = Number(gradle.match(pattern)?.[2] ?? 0); + const versionCode = Math.max(local, await fetchVersionCode(packageName)) + 1; + fs.writeFileSync(gradlePath, gradle.replace(pattern, `$1${versionCode}`)); + fs.writeFileSync( + process.env.STORE_BUILD_METADATA ?? 'store-build.json', + JSON.stringify({ packageName, versionCode }) + ); +} -let gradleVersionCode = 0; +async function submit() { + const metadata = process.env.STORE_BUILD_METADATA; + const bundle = process.env.STORE_BUNDLE; + const receipt = process.env.STORE_RECEIPT; + if (!metadata || !bundle || !receipt) { + throw new Error( + 'STORE_BUILD_METADATA, STORE_BUNDLE and STORE_RECEIPT are required' + ); + } + const operation = { + operationRunId: Number(process.env.STORE_OPERATION_RUN_ID), + operationAttempt: Number(process.env.STORE_OPERATION_ATTEMPT), + }; + if ( + !Number.isSafeInteger(operation.operationRunId) || + operation.operationRunId <= 0 || + !Number.isSafeInteger(operation.operationAttempt) || + operation.operationAttempt <= 0 + ) { + throw new Error( + 'A positive store operation run ID and attempt are required' + ); + } + const candidate: { packageName: string; versionCode: number } = JSON.parse( + fs.readFileSync(metadata, 'utf8') + ); + if ( + candidate.packageName !== packageName || + !Number.isSafeInteger(candidate.versionCode) + ) { + throw new Error('Invalid Google Play candidate metadata'); + } + const editId = await createEdit(packageName); + const tracks = await publisher.edits.tracks.list({ packageName, editId }); + const submitted = tracks.data.tracks?.some(track => + track.releases?.some(release => + release.versionCodes?.includes(String(candidate.versionCode)) + ) + ); + if (process.env.STORE_QUERY_ONLY === '1') { + await publisher.edits.delete({ packageName, editId }); + if (!submitted && process.env.STORE_ALLOW_ABSENT !== '1') { + throw new Error( + 'No visible store result; the previous submission step ran, so its result remains unresolved' + ); + } + fs.writeFileSync( + receipt, + JSON.stringify({ + ...candidate, + store: 'google-play', + status: submitted ? 'submitted' : 'not-submitted', + ...operation, + }) + ); + return; + } + if (submitted) { + await publisher.edits.delete({ packageName, editId }); + } else { + const uploaded = await publisher.edits.bundles.list({ + packageName, + editId, + }); + if ( + !uploaded.data.bundles?.some( + item => item.versionCode === candidate.versionCode + ) + ) { + const result = await publisher.edits.bundles.upload({ + packageName, + editId, + media: { + mimeType: 'application/octet-stream', + body: fs.createReadStream(bundle), + }, + }); + if (result.data.versionCode !== candidate.versionCode) { + throw new Error( + 'Uploaded bundle version does not match the prepared candidate' + ); + } + } + await publisher.edits.tracks.update({ + packageName, + editId, + track: 'internal', + requestBody: { + releases: [ + { status: 'draft', versionCodes: [String(candidate.versionCode)] }, + ], + }, + }); + await publisher.edits.commit({ packageName, editId }); + } + fs.writeFileSync( + receipt, + JSON.stringify({ + ...candidate, + ...operation, + store: 'google-play', + status: 'submitted', + }) + ); +} -const gradleFile = fs.readFileSync(gradlePath, 'utf8'); -const matched = gradleFile.match(versionCodeRegexPattern); - -const remoteVersion = await fetchVersionCode('app.affine.pro'); - -gradleVersionCode = parseInt(matched?.[2] || '0'); -gradleVersionCode = isNaN(gradleVersionCode) ? 0 : gradleVersionCode; -const versionCode = Math.max(gradleVersionCode, remoteVersion) + 1; - -fs.writeFileSync( - gradlePath, - gradleFile.replace(versionCodeRegexPattern, `$1 ${versionCode}`) -); +if (process.argv.includes('submit')) { + await submit(); +} else { + await bump(); +} diff --git a/tools/playstore-auto-bump/package.json b/tools/playstore-auto-bump/package.json index a39a874a9c..362d801ed2 100644 --- a/tools/playstore-auto-bump/package.json +++ b/tools/playstore-auto-bump/package.json @@ -6,7 +6,8 @@ "description": "Automatically bump the versionCode and versionName of an Android app from the Google Play Store versions", "main": "index.ts", "scripts": { - "bump": "r ./index.ts" + "bump": "r ./index.ts", + "submit": "r ./index.ts submit" }, "dependencies": { "@affine-tools/cli": "workspace:*", diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 369e95b637..bf0a4aa991 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -1423,7 +1423,7 @@ export const PackageList = [ { location: 'tools/cli', name: '@affine-tools/cli', - workspaceDependencies: ['tools/utils', 'packages/common/s3-compat'], + workspaceDependencies: ['tools/utils'], }, { location: 'tools/commitlint', diff --git a/yarn.lock b/yarn.lock index 4d8b09bfcc..c4dc9b0615 100644 --- a/yarn.lock +++ b/yarn.lock @@ -106,7 +106,6 @@ __metadata: resolution: "@affine-tools/cli@workspace:tools/cli" dependencies: "@affine-tools/utils": "workspace:*" - "@affine/s3-compat": "workspace:*" "@napi-rs/simple-git": "npm:^1.0.0" "@rspack/core": "npm:^2.0.8" "@rspack/dev-server": "npm:^2.1.0" @@ -114,7 +113,6 @@ __metadata: "@swc/core": "npm:^1.10.1" "@tailwindcss/postcss": "npm:^4.3.0" "@types/lodash-es": "npm:^4.17.12" - "@types/mime-types": "npm:^3.0.0" "@types/node": "npm:^22.0.0" "@vanilla-extract/webpack-plugin": "npm:^2.3.15" autoprefixer: "npm:^10.4.20" @@ -124,7 +122,6 @@ __metadata: inquirer: "npm:^12.3.0" jsonc-parser: "npm:^3.3.1" lodash-es: "npm:^4.17.23" - mime-types: "npm:^3.0.0" oxfmt: "npm:0.63.0" postcss: "npm:^8.5.18" postcss-loader: "npm:^8.2.1" @@ -563,6 +560,8 @@ __metadata: "@electron-forge/plugin-auto-unpack-natives": "npm:^7.11.1" "@electron-forge/plugin-fuses": "npm:^7.11.1" "@electron-forge/shared-types": "npm:^7.11.1" + "@electron/notarize": "npm:2.5.0" + "@electron/osx-sign": "npm:1.3.3" "@reforged/maker-appimage": "npm:^5.2.0" "@sentry/electron": "npm:^7.20.1" "@sentry/esbuild-plugin": "npm:^5.3.0" @@ -14650,13 +14649,6 @@ __metadata: languageName: node linkType: hard -"@types/mime-types@npm:^3.0.0": - version: 3.0.0 - resolution: "@types/mime-types@npm:3.0.0" - checksum: 10/64745608ff00e2cd1b7ee08705ec02a07bae6c34e2a70d2ad40527836220b9b9c5618b4cb32bdd4dc74d409ff5b069b73a4d85ff1636544ff9423409779ff8c7 - languageName: node - linkType: hard - "@types/mime@npm:^1": version: 1.3.5 resolution: "@types/mime@npm:1.3.5" @@ -17111,19 +17103,19 @@ __metadata: linkType: hard "body-parser@npm:^2.2.1, body-parser@npm:^2.2.2": - version: 2.3.0 - resolution: "body-parser@npm:2.3.0" + version: 2.4.0 + resolution: "body-parser@npm:2.4.0" dependencies: bytes: "npm:^3.1.2" - content-type: "npm:^2.0.0" + content-type: "npm:^2.1.0" debug: "npm:^4.4.3" http-errors: "npm:^2.0.1" - iconv-lite: "npm:^0.7.2" + iconv-lite: "npm:^0.7.3" on-finished: "npm:^2.4.1" - qs: "npm:^6.15.2" + qs: "npm:^6.16.1" raw-body: "npm:^3.0.2" type-is: "npm:^2.1.0" - checksum: 10/94f741ff525358dd0d47f0a042936dc61b9d2e348ff1228c5ffd79f1902c673a317bbb9495b3f7d1db4483befe6ffed1c42bc4cdd8da8f41690530810cfe4dad + checksum: 10/e34971abf7051529d4fae37d3714382e7fa8198bc969f1ec4932af48aef53bba71523545c034ce04fa808d3b15e0d63849e6e93b4f1131d65227e1f414437f68 languageName: node linkType: hard @@ -18494,10 +18486,10 @@ __metadata: languageName: node linkType: hard -"content-type@npm:^2.0.0": - version: 2.0.0 - resolution: "content-type@npm:2.0.0" - checksum: 10/0bbb276b790ba7e86c479c7d69fae1861b2e908ff3ce2cb01975b516f93eede2216d242902ed6c5b15cd554014611ce9dfdcf51cd35b16569e45a979e50d0cef +"content-type@npm:^2.0.0, content-type@npm:^2.1.0": + version: 2.1.0 + resolution: "content-type@npm:2.1.0" + checksum: 10/8aac907922538f430d8bfab1d8fe2fbdc1eb890a0fa7a619f712db4bfefe24be951ba464f972007df64840c775bf2db284ca8223547ac5317ad9ab2ee8e6501b languageName: node linkType: hard @@ -19857,14 +19849,14 @@ __metadata: linkType: hard "dompurify@npm:^3.3.3, dompurify@npm:^3.4.11": - version: 3.4.16 - resolution: "dompurify@npm:3.4.16" + version: 3.4.17 + resolution: "dompurify@npm:3.4.17" dependencies: "@types/trusted-types": "npm:^2.0.7" dependenciesMeta: "@types/trusted-types": optional: true - checksum: 10/a571d0e2bede4f739666afd640398a86854abd27c71ce0724910249e2fd2ca7ddb8098a9ab579d3a5b39058419af694421f8fe859a81fd6d8acda16019a8ab9d + checksum: 10/ae03c068bd351d1201a24a9667cb302c65939f45a0c357edec8b84133108e19ac74f355c2d8b0f30baadadd7f0be2db300d269aa80d3ee15872712ca9ff4e1a8 languageName: node linkType: hard @@ -21242,9 +21234,9 @@ __metadata: linkType: hard "fast-uri@npm:^3.0.1": - version: 3.1.8 - resolution: "fast-uri@npm:3.1.8" - checksum: 10/d3dfa29fd2e5fa2371833699f3f78c975b79f65e52e0ce37604db10dffb85b5b448b5075e93453272cb95fb1c6a759495164e60649ffd69e59ec43f20d44f791 + version: 3.1.9 + resolution: "fast-uri@npm:3.1.9" + checksum: 10/4276cf58661e68b4fefcbc3308a2c8429e5664d2a77b393a79c1cdbe9e5ba80617d27c7ce6685194ae40e4816df73dbd72a0324e5012d94a7e9cf85ee711f658 languageName: node linkType: hard @@ -22856,7 +22848,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": +"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.3, iconv-lite@npm:~0.7.0": version: 0.7.3 resolution: "iconv-lite@npm:0.7.3" dependencies: @@ -23154,9 +23146,9 @@ __metadata: linkType: hard "ip-address@npm:^10.1.1": - version: 10.7.3 - resolution: "ip-address@npm:10.7.3" - checksum: 10/04655aa25ad682bb26ecb82281c9d3ecd99da8ecde9c95146f0a96685de90a359432d905bf6c53b5732fed0bfb1f93763cd3c91c3f266eec96cc41bfea987197 + version: 10.7.4 + resolution: "ip-address@npm:10.7.4" + checksum: 10/89375f1fa00e5bb4e724498886e06d870c1cc0abbfa9df8cb5ce80f3f179ee49feb0ca96ec54a1512c2303302719975a2ad96e3a8fcc13b4fd836d7f461d3d95 languageName: node linkType: hard @@ -28741,13 +28733,13 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.11.2, qs@npm:^6.14.0, qs@npm:^6.15.2, qs@npm:^6.7.0": - version: 6.16.0 - resolution: "qs@npm:6.16.0" +"qs@npm:^6.11.2, qs@npm:^6.14.0, qs@npm:^6.16.1, qs@npm:^6.7.0": + version: 6.16.1 + resolution: "qs@npm:6.16.1" dependencies: es-define-property: "npm:^1.0.1" side-channel: "npm:^1.1.1" - checksum: 10/7fbf9c2eb9c9bbd8997700caecc4b057ac259635873deefd5027fb26d06d79426cf468da55da87ffd147a24e93d3641352d8ac6664bd84688ad5b2f0b8d04828 + checksum: 10/e33e0a5ceecb7bef81ea1f855365482045f55ba55278f40b8dbf9ee18eae8ef82d44246fe4e4009d6b6f102cbf2885a34b88c4272536cd628cde40b4ac9912c2 languageName: node linkType: hard @@ -32114,16 +32106,16 @@ __metadata: linkType: hard "undici@npm:^6.25.0": - version: 6.29.0 - resolution: "undici@npm:6.29.0" - checksum: 10/fd4969cfc846cb4da7f7b1b225536d629e17b17b4a1913287f1264f30c855b9eb65a6da6cde46ba1ddfe42f113a3969ab2697029c2b920401be2b7b3b40abef8 + version: 6.29.1 + resolution: "undici@npm:6.29.1" + checksum: 10/92acb64ec190fe3499dec69830fdfc48e28f6f19627bad7563cfd57c37b4e95379c12b8ac15a2e6a8739eaac358f17503a229c160d3dc8774141b1e270d8aa57 languageName: node linkType: hard "undici@npm:^7.24.4": - version: 7.30.0 - resolution: "undici@npm:7.30.0" - checksum: 10/eb9e4bb5f1b4c35cb934c0d04fe676b62c3be06fb3be4a256ff73699bd77abc3d41f262597a4debd41f334c95ca3ea6e763268c3768882c3fc88a5d33234b8f6 + version: 7.30.1 + resolution: "undici@npm:7.30.1" + checksum: 10/306a28e4c715652a0e2cd59bcc98cf8b6083b989151054fafaac27bdab6f2bc7e0f33f5847f72f5ba257971226372d04705e0e984b26b42a03271d5a35eb99b1 languageName: node linkType: hard