From 33fd141762a8f42c7ee97056860180cac513ff04 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sat, 30 May 2026 17:46:27 +0800 Subject: [PATCH 01/17] feat(github): store token scopes Assisted-by: Codex --- .../sql-token-persistence.integration.js | 61 ++++++++++++++++++- core/token-pooling/sql-token-persistence.js | 20 +++--- .../sql-token-persistence.spec.js | 50 +++++++++++++++ migrations/1780134030603_add-token-scopes.cjs | 11 ++++ services/github/github-api-provider.js | 8 +-- services/github/github-constellation.js | 4 +- 6 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 core/token-pooling/sql-token-persistence.spec.js create mode 100644 migrations/1780134030603_add-token-scopes.cjs diff --git a/core/token-pooling/sql-token-persistence.integration.js b/core/token-pooling/sql-token-persistence.integration.js index 2febee7483924..c9c98cafdc2ac 100644 --- a/core/token-pooling/sql-token-persistence.integration.js +++ b/core/token-pooling/sql-token-persistence.integration.js @@ -54,17 +54,38 @@ describe('SQL token persistence', function () { const initialTokens = ['a', 'b', 'c'].map(char => char.repeat(40)) beforeEach(async function () { - initialTokens.forEach(async token => { + for (const token of initialTokens) { await pool.query( `INSERT INTO pg_temp.${tableName} (token) VALUES ($1::text);`, [token], ) - }) + } }) it('loads the contents', async function () { const tokens = await persistence.initialize(pool) - expect(tokens.sort()).to.deep.equal(initialTokens) + expect(tokens.map(({ token }) => token).sort()).to.deep.equal( + initialTokens, + ) + expect(tokens.map(({ scopes }) => scopes)).to.deep.equal([ + null, + null, + null, + ]) + }) + + it('loads token scopes', async function () { + const scopedToken = 'd'.repeat(40) + await pool.query( + `INSERT INTO pg_temp.${tableName} (token, scopes) VALUES ($1::text, $2::text[]);`, + [scopedToken, ['read:packages', 'read:user']], + ) + + const tokens = await persistence.initialize(pool) + expect(tokens).to.deep.include({ + token: scopedToken, + scopes: ['read:packages', 'read:user'], + }) }) context('when tokens are added', function () { @@ -82,6 +103,40 @@ describe('SQL token persistence', function () { const savedTokens = result.rows.map(row => row.token) expect(savedTokens.sort()).to.deep.equal(expected) }) + + it('saves token scopes', async function () { + const newToken = 'e'.repeat(40) + + await persistence.initialize(pool) + await persistence.noteTokenAdded(newToken, { + scopes: ['read:packages'], + }) + + const result = await pool.query( + `SELECT token, scopes FROM pg_temp.${tableName} WHERE token=$1::text;`, + [newToken], + ) + expect(result.rows).to.deep.equal([ + { token: newToken, scopes: ['read:packages'] }, + ]) + }) + + it('updates token scopes when a known token is re-added with scopes', async function () { + const knownToken = initialTokens[0] + + await persistence.initialize(pool) + await persistence.noteTokenAdded(knownToken, { + scopes: ['read:packages'], + }) + + const result = await pool.query( + `SELECT token, scopes FROM pg_temp.${tableName} WHERE token=$1::text;`, + [knownToken], + ) + expect(result.rows).to.deep.equal([ + { token: knownToken, scopes: ['read:packages'] }, + ]) + }) }) context('when tokens are removed', function () { diff --git a/core/token-pooling/sql-token-persistence.js b/core/token-pooling/sql-token-persistence.js index 610df1b973c09..2bd4e196262ba 100644 --- a/core/token-pooling/sql-token-persistence.js +++ b/core/token-pooling/sql-token-persistence.js @@ -16,9 +16,9 @@ export default class SqlTokenPersistence { this.pool = new pg.Pool({ connectionString: this.url }) } const result = await this.pool.query( - `SELECT token FROM ${this.table} ORDER BY RANDOM();`, + `SELECT token, scopes FROM ${this.table} ORDER BY RANDOM();`, ) - return result.rows.map(row => row.token) + return result.rows.map(({ token, scopes }) => ({ token, scopes })) } async stop() { @@ -27,10 +27,16 @@ export default class SqlTokenPersistence { } } - async onTokenAdded(token) { + async onTokenAdded(token, { scopes } = {}) { return await this.pool.query( - `INSERT INTO ${this.table} (token) VALUES ($1::text) ON CONFLICT (token) DO NOTHING;`, - [token], + ` + INSERT INTO ${this.table} (token, scopes) + VALUES ($1::text, $2::text[]) + ON CONFLICT (token) DO UPDATE + SET scopes = EXCLUDED.scopes + WHERE EXCLUDED.scopes IS NOT NULL; + `, + [token, scopes], ) } @@ -41,9 +47,9 @@ export default class SqlTokenPersistence { ) } - async noteTokenAdded(token) { + async noteTokenAdded(token, data) { try { - await this.onTokenAdded(token) + await this.onTokenAdded(token, data) } catch (e) { log.error(e) } diff --git a/core/token-pooling/sql-token-persistence.spec.js b/core/token-pooling/sql-token-persistence.spec.js new file mode 100644 index 0000000000000..d22b55d54f36b --- /dev/null +++ b/core/token-pooling/sql-token-persistence.spec.js @@ -0,0 +1,50 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import SqlTokenPersistence from './sql-token-persistence.js' + +describe('SQL token persistence unit', function () { + const tableName = 'github_user_tokens' + + let pool + let persistence + + beforeEach(function () { + pool = { query: sinon.stub() } + persistence = new SqlTokenPersistence({ + url: 'postgres://example.test/shields', + table: tableName, + }) + }) + + it('loads token scopes', async function () { + pool.query.resolves({ + rows: [ + { token: 'unscoped-token', scopes: null }, + { token: 'scoped-token', scopes: ['read:packages'] }, + ], + }) + + const tokens = await persistence.initialize(pool) + + expect(tokens).to.deep.equal([ + { token: 'unscoped-token', scopes: null }, + { token: 'scoped-token', scopes: ['read:packages'] }, + ]) + sinon.assert.calledOnceWithExactly( + pool.query, + `SELECT token, scopes FROM ${tableName} ORDER BY RANDOM();`, + ) + }) + + it('saves token scopes', async function () { + pool.query.resolves({ rows: [] }) + await persistence.initialize(pool) + + await persistence.onTokenAdded('scoped-token', { + scopes: ['read:packages'], + }) + + const [, params] = pool.query.secondCall.args + expect(params).to.deep.equal(['scoped-token', ['read:packages']]) + }) +}) diff --git a/migrations/1780134030603_add-token-scopes.cjs b/migrations/1780134030603_add-token-scopes.cjs new file mode 100644 index 0000000000000..9a820b2133850 --- /dev/null +++ b/migrations/1780134030603_add-token-scopes.cjs @@ -0,0 +1,11 @@ +exports.shorthands = undefined + +exports.up = pgm => { + pgm.addColumn('github_user_tokens', { + scopes: { type: 'text[]' }, + }) +} + +exports.down = pgm => { + pgm.dropColumn('github_user_tokens', 'scopes') +} diff --git a/services/github/github-api-provider.js b/services/github/github-api-provider.js index eceb743ea44bb..f44339fd098a3 100644 --- a/services/github/github-api-provider.js +++ b/services/github/github-api-provider.js @@ -65,11 +65,11 @@ class GithubApiProvider { this.restApiVersion = restApiVersion } - addToken(tokenString) { + addToken(tokenString, data) { if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) { - this.standardTokens.add(tokenString) - this.searchTokens.add(tokenString) - this.graphqlTokens.add(tokenString) + this.standardTokens.add(tokenString, data) + this.searchTokens.add(tokenString, data) + this.graphqlTokens.add(tokenString, data) } else { throw Error('When not using a token pool, do not provide tokens') } diff --git a/services/github/github-constellation.js b/services/github/github-constellation.js index bfc887b1f82d5..bc443075ab106 100644 --- a/services/github/github-constellation.js +++ b/services/github/github-constellation.js @@ -92,8 +92,8 @@ class GithubConstellation { log.error(e) } - tokens.forEach(tokenString => { - this.apiProvider.addToken(tokenString) + tokens.forEach(({ token, scopes }) => { + this.apiProvider.addToken(token, { scopes }) }) if (this.oauthHelper.isConfigured) { From e361c35fa762cd27cfd84a4effb554effd7f4d77 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sat, 30 May 2026 17:58:55 +0800 Subject: [PATCH 02/17] perf(github): index token scopes Assisted-by: Codex --- migrations/1780134030603_add-token-scopes.cjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrations/1780134030603_add-token-scopes.cjs b/migrations/1780134030603_add-token-scopes.cjs index 9a820b2133850..bc28c1613bb8b 100644 --- a/migrations/1780134030603_add-token-scopes.cjs +++ b/migrations/1780134030603_add-token-scopes.cjs @@ -4,8 +4,10 @@ exports.up = pgm => { pgm.addColumn('github_user_tokens', { scopes: { type: 'text[]' }, }) + pgm.createIndex('github_user_tokens', 'scopes', { method: 'gin' }) } exports.down = pgm => { + pgm.dropIndex('github_user_tokens', 'scopes') pgm.dropColumn('github_user_tokens', 'scopes') } From 8ca940894aa9fa53716d0bfb5049c09740fb7cb2 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 08:56:44 +0800 Subject: [PATCH 03/17] feat(github): persist accepted token scopes Assisted-by: Codex --- services/github/auth/acceptor.js | 5 +++-- services/github/auth/acceptor.spec.js | 20 +++++++++++++++++++- services/github/github-constellation.js | 9 +++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index ac7ec88e949e7..fae83a406b36e 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -58,10 +58,11 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { return end('The GitHub OAuth token could not be parsed.') } - const { access_token: token } = content + const { access_token: token, scope = '' } = content if (!token) { return end('The GitHub OAuth process did not return a user token.') } + const scopes = scope.split(',').filter(Boolean) ask.res.setHeader('Content-Type', 'text/html') end( @@ -76,7 +77,7 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { '

Back to the website

', ) - onTokenAccepted(token) + onTokenAccepted(token, { scopes }) }) } diff --git a/services/github/auth/acceptor.spec.js b/services/github/auth/acceptor.spec.js index dd22769adbaa5..798d09726ce49 100644 --- a/services/github/auth/acceptor.spec.js +++ b/services/github/auth/acceptor.spec.js @@ -5,6 +5,7 @@ import sinon from 'sinon' import portfinder from 'portfinder' import qs from 'qs' import nock from 'nock' +import '../../../core/register-chai-plugins.spec.js' import got from '../../../core/got-test-client.js' import GithubConstellation from '../github-constellation.js' import { setRoutes } from './acceptor.js' @@ -73,8 +74,10 @@ describe('Github token acceptor', function () { context('a code is provided', function () { let scope + let tokenResponse beforeEach(function () { nock.enableNetConnect(/127\.0\.0\.1/) + tokenResponse = { access_token: fakeAccessToken } scope = nock('https://github.com') .post('/login/oauth/access_token') @@ -83,7 +86,7 @@ describe('Github token acceptor', function () { expect(parsedBody.client_id).to.equal(fakeClientId) expect(parsedBody.client_secret).to.equal(fakeClientSecret) expect(parsedBody.code).to.equal(fakeCode) - return [200, qs.stringify({ access_token: fakeAccessToken })] + return [200, qs.stringify(tokenResponse)] }) }) @@ -118,6 +121,21 @@ describe('Github token acceptor', function () { ).to.be.true expect(onTokenAccepted).to.have.been.calledWith(fakeAccessToken) }) + + it('should pass token scopes from the OAuth response', async function () { + tokenResponse.scope = 'read:packages,read:user' + + const form = new FormData() + form.append('code', fakeCode) + + await got.post(`${baseUrl}/github-auth/done`, { + body: form, + }) + + expect(onTokenAccepted).to.have.been.calledWith(fakeAccessToken, { + scopes: ['read:packages', 'read:user'], + }) + }) }) }) }) diff --git a/services/github/github-constellation.js b/services/github/github-constellation.js index bc443075ab106..20303b1572650 100644 --- a/services/github/github-constellation.js +++ b/services/github/github-constellation.js @@ -100,19 +100,20 @@ class GithubConstellation { setAcceptorRoutes({ server, authHelper: this.oauthHelper, - onTokenAccepted: tokenString => this.onTokenAdded(tokenString), + onTokenAccepted: (tokenString, data) => + this.onTokenAdded(tokenString, data), }) } } - onTokenAdded(tokenString) { + onTokenAdded(tokenString, data) { if (!this.persistence) { throw Error('Token persistence is not configured') } - this.apiProvider.addToken(tokenString) + this.apiProvider.addToken(tokenString, data) process.nextTick(async () => { try { - await this.persistence.noteTokenAdded(tokenString) + await this.persistence.noteTokenAdded(tokenString, data) } catch (e) { log.error(e) } From df0b2940ac5fb72320de514fa653ef4a3f2c6178 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 09:23:03 +0800 Subject: [PATCH 04/17] docs(github): clarify OAuth scope capture Assisted-by: Codex --- services/github/auth/acceptor.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index fae83a406b36e..6efac017139a8 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -14,6 +14,10 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { // it's not setting a bad example. client_id: authHelper._user, redirect_uri: `${baseUrl}/github-auth/done`, + // Do not add OAuth scopes here as part of scope persistence plumbing. + // Requesting scopes changes the user consent surface and should be a + // separate change. Until then, persisted scopes only reflect what GitHub + // returns for the current OAuth request. }) ask.res.setHeader( 'Location', From 0b9af464cbc851942a3919b5dc86a3cc1694de22 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 09:24:50 +0800 Subject: [PATCH 05/17] docs(github): shorten OAuth scope note Assisted-by: Codex --- services/github/auth/acceptor.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index 6efac017139a8..e7317391dd86f 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -14,10 +14,8 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { // it's not setting a bad example. client_id: authHelper._user, redirect_uri: `${baseUrl}/github-auth/done`, - // Do not add OAuth scopes here as part of scope persistence plumbing. - // Requesting scopes changes the user consent surface and should be a - // separate change. Until then, persisted scopes only reflect what GitHub - // returns for the current OAuth request. + // Requesting scopes changes the OAuth consent surface, so keep that + // separate from persisting whatever scopes GitHub returns here. }) ask.res.setHeader( 'Location', From 0799075d655291eef11ca7d41ce47c91a06e5b8a Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 09:26:35 +0800 Subject: [PATCH 06/17] docs(github): clarify default OAuth scopes Assisted-by: Codex --- services/github/auth/acceptor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index e7317391dd86f..865d0789c4cb2 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -14,8 +14,9 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { // it's not setting a bad example. client_id: authHelper._user, redirect_uri: `${baseUrl}/github-auth/done`, - // Requesting scopes changes the OAuth consent surface, so keep that - // separate from persisting whatever scopes GitHub returns here. + // Keep this authorization unscoped by default. Features that need extra + // GitHub permissions should request them explicitly because users will + // see a broader OAuth consent prompt. }) ask.res.setHeader( 'Location', From f494d2275c2c1ade10fe9ee6099e7c6a7afb88ea Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 09:33:05 +0800 Subject: [PATCH 07/17] docs(github): link OAuth scope discussion Assisted-by: Codex --- services/github/auth/acceptor.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index 865d0789c4cb2..8f966df86f2b9 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -14,9 +14,9 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { // it's not setting a bad example. client_id: authHelper._user, redirect_uri: `${baseUrl}/github-auth/done`, - // Keep this authorization unscoped by default. Features that need extra - // GitHub permissions should request them explicitly because users will - // see a broader OAuth consent prompt. + // This OAuth flow does not request extra scopes yet. Adding scopes changes + // the user consent prompt and is tracked in + // https://github.com/badges/shields/issues/4169. }) ask.res.setHeader( 'Location', From 5e201d50a137863b707e930fe734fa456da2aaeb Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 22:24:15 +0800 Subject: [PATCH 08/17] docs(github): mark OAuth scopes as TODO Assisted-by: Codex --- services/github/auth/acceptor.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index 8f966df86f2b9..f8b4e086e0870 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -14,9 +14,9 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { // it's not setting a bad example. client_id: authHelper._user, redirect_uri: `${baseUrl}/github-auth/done`, - // This OAuth flow does not request extra scopes yet. Adding scopes changes - // the user consent prompt and is tracked in - // https://github.com/badges/shields/issues/4169. + // TODO: Request the OAuth scopes needed for scoped GitHub API features. + // Remove this TODO once this authorize URL includes a `scope` parameter. + // See https://github.com/badges/shields/issues/4169. }) ask.res.setHeader( 'Location', From 64b6e25b075e8ea052556b5881de80b166f5f2d0 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 22:48:56 +0800 Subject: [PATCH 09/17] fix(token-pool): refresh existing token data Assisted-by: Codex --- core/token-pooling/token-pool.js | 31 +++++++++++++++++++++++---- core/token-pooling/token-pool.spec.js | 29 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/core/token-pooling/token-pool.js b/core/token-pooling/token-pool.js index 441daedb88a8c..6ccb136dc4c88 100644 --- a/core/token-pooling/token-pool.js +++ b/core/token-pooling/token-pool.js @@ -84,6 +84,15 @@ class Token { return this._usesRemaining - 1 } + /** + * Update user-provided data associated with the token. + * + * @param {*} data reserved for future use + */ + updateData(data) { + this._data = data + } + /** * Update the uses remaining and next reset time for a token. * @@ -128,6 +137,13 @@ class Token { this._isValid = false } + /** + * Indicate that the token should be used again. + */ + validate() { + this._isValid = true + } + /** * Freeze the uses remaining and next reset values. Helpful for keeping * stable ordering for a valid priority queue. @@ -184,8 +200,8 @@ class TokenPool { this.currentBatch = { currentToken: null, remaining: 0 } - // A set of IDs used for deduplication. - this.tokenIds = new Set() + // A map of IDs to tokens, used for deduplication and metadata updates. + this.tokensById = new Map() // See discussion on the FIFO and priority queues in `next()`. this.fifoQueue = [] @@ -216,15 +232,22 @@ class TokenPool { * @returns {boolean} Was the token added to the pool? */ add(id, data, usesRemaining, nextReset) { - if (this.tokenIds.has(id)) { + const existingToken = this.tokensById.get(id) + if (existingToken) { + const wasInvalid = !existingToken.isValid + existingToken.updateData(data) + existingToken.validate() + if (wasInvalid) { + this.fifoQueue.push(existingToken) + } return false } - this.tokenIds.add(id) usesRemaining = usesRemaining === undefined ? this.batchSize : usesRemaining nextReset = nextReset === undefined ? Token.nextResetNever : nextReset const token = new Token(id, data, usesRemaining, nextReset) + this.tokensById.set(id, token) this.fifoQueue.push(token) return true diff --git a/core/token-pooling/token-pool.spec.js b/core/token-pooling/token-pool.spec.js index 95208feaa046f..ead9c6cbc6845 100644 --- a/core/token-pooling/token-pool.spec.js +++ b/core/token-pooling/token-pool.spec.js @@ -35,6 +35,35 @@ describe('The token pool', function () { ) }) + it('updates data when an existing token is added again', function () { + const tokenPool = new TokenPool() + + expect(tokenPool.add('token', { scopes: [] })).to.equal(true) + expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal( + false, + ) + + expect(tokenPool.next().data).to.deep.equal({ + scopes: ['read:packages'], + }) + }) + + it('revalidates invalid tokens when they are added again', function () { + const tokenPool = new TokenPool() + expect(tokenPool.add('token', { scopes: [] })).to.equal(true) + + tokenPool.next().invalidate() + expectPoolToBeExhausted(tokenPool) + + expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal( + false, + ) + + const token = tokenPool.next() + expect(token.id).to.equal('token') + expect(token.data).to.deep.equal({ scopes: ['read:packages'] }) + }) + context('tokens are marked exhausted immediately', function () { it('should be exhausted', function () { ids.forEach(() => { From f7d4f395cca61994aa177e29e0bbcb58d999b38b Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 23:05:49 +0800 Subject: [PATCH 10/17] fix(token-pool): avoid duplicate revalidated tokens Assisted-by: Codex:gpt-5.5 --- core/token-pooling/token-pool.js | 6 +++++- core/token-pooling/token-pool.spec.js | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/token-pooling/token-pool.js b/core/token-pooling/token-pool.js index 6ccb136dc4c88..d5b542f79a336 100644 --- a/core/token-pooling/token-pool.js +++ b/core/token-pooling/token-pool.js @@ -237,7 +237,11 @@ class TokenPool { const wasInvalid = !existingToken.isValid existingToken.updateData(data) existingToken.validate() - if (wasInvalid) { + const isQueued = this.fifoQueue.includes(existingToken) + const isCurrent = + this.currentBatch.token === existingToken && + this.currentBatch.remaining > 0 + if (wasInvalid && !isQueued && !isCurrent) { this.fifoQueue.push(existingToken) } return false diff --git a/core/token-pooling/token-pool.spec.js b/core/token-pooling/token-pool.spec.js index ead9c6cbc6845..80fe7a5333d28 100644 --- a/core/token-pooling/token-pool.spec.js +++ b/core/token-pooling/token-pool.spec.js @@ -64,6 +64,20 @@ describe('The token pool', function () { expect(token.data).to.deep.equal({ scopes: ['read:packages'] }) }) + it('does not duplicate queued tokens when they are revalidated', function () { + const tokenPool = new TokenPool({ batchSize: 2 }) + tokenPool.add('first') + tokenPool.add('second') + + tokenPool.next().invalidate() + tokenPool.add('first') + + expect(tokenPool.next().id).to.equal('first') + expect(tokenPool.next().id).to.equal('second') + expect(tokenPool.next().id).to.equal('second') + expect(tokenPool.next().id).to.equal('first') + }) + context('tokens are marked exhausted immediately', function () { it('should be exhausted', function () { ids.forEach(() => { From 946a2f4d0888faac51667579d3e99ffccb682b50 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 31 May 2026 23:45:10 +0800 Subject: [PATCH 11/17] fix(token-pool): remove revalidated tokens from priority queue Assisted-by: Codex:gpt-5.5 --- core/token-pooling/token-pool.js | 14 ++++++++++++++ core/token-pooling/token-pool.spec.js | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/core/token-pooling/token-pool.js b/core/token-pooling/token-pool.js index d5b542f79a336..f0a5c1e16704f 100644 --- a/core/token-pooling/token-pool.js +++ b/core/token-pooling/token-pool.js @@ -219,6 +219,16 @@ class TokenPool { return second.nextReset - first.nextReset } + _removeFromPriorityQueue(token) { + const priorityQueue = new PriorityQueue(this.constructor.compareTokens) + this.priorityQueue.forEach(queuedToken => { + if (queuedToken !== token) { + priorityQueue.enq(queuedToken) + } + }) + this.priorityQueue = priorityQueue + } + /** * Add a token with user-provided ID and data. * @@ -236,6 +246,10 @@ class TokenPool { if (existingToken) { const wasInvalid = !existingToken.isValid existingToken.updateData(data) + if (wasInvalid) { + this._removeFromPriorityQueue(existingToken) + existingToken.unfreeze() + } existingToken.validate() const isQueued = this.fifoQueue.includes(existingToken) const isCurrent = diff --git a/core/token-pooling/token-pool.spec.js b/core/token-pooling/token-pool.spec.js index 80fe7a5333d28..f1678334cee7b 100644 --- a/core/token-pooling/token-pool.spec.js +++ b/core/token-pooling/token-pool.spec.js @@ -78,6 +78,29 @@ describe('The token pool', function () { expect(tokenPool.next().id).to.equal('first') }) + it('does not duplicate exhausted tokens when they are revalidated', function () { + const tokenPool = new TokenPool() + tokenPool.add('first') + tokenPool.add('second') + + const first = tokenPool.next() + first.update(0, Token.nextResetNever) + tokenPool.next() + + first.invalidate() + tokenPool.add('first') + + const { allTokenDebugInfo } = tokenPool.serializeDebugInfo({ + sanitize: false, + }) + const tokenIds = allTokenDebugInfo.map(({ id }) => id) + + expect(tokenIds).to.have.members(['first', 'second']) + expect( + allTokenDebugInfo.find(({ id }) => id === 'first').isFrozen, + ).to.equal(false) + }) + context('tokens are marked exhausted immediately', function () { it('should be exhausted', function () { ids.forEach(() => { From ecdb644aebbd92d0ce9648c32b358044a2e28349 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 00:35:03 +0800 Subject: [PATCH 12/17] fix(token-pool): preserve token metadata on re-add Assisted-by: Codex:gpt-5.5 --- core/token-pooling/token-pool.js | 4 +++- core/token-pooling/token-pool.spec.js | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/core/token-pooling/token-pool.js b/core/token-pooling/token-pool.js index f0a5c1e16704f..79de6aec3e633 100644 --- a/core/token-pooling/token-pool.js +++ b/core/token-pooling/token-pool.js @@ -245,7 +245,9 @@ class TokenPool { const existingToken = this.tokensById.get(id) if (existingToken) { const wasInvalid = !existingToken.isValid - existingToken.updateData(data) + if (data !== undefined) { + existingToken.updateData(data) + } if (wasInvalid) { this._removeFromPriorityQueue(existingToken) existingToken.unfreeze() diff --git a/core/token-pooling/token-pool.spec.js b/core/token-pooling/token-pool.spec.js index f1678334cee7b..8016e817e8735 100644 --- a/core/token-pooling/token-pool.spec.js +++ b/core/token-pooling/token-pool.spec.js @@ -48,6 +48,17 @@ describe('The token pool', function () { }) }) + it('preserves existing data when an existing token is added again without data', function () { + const tokenPool = new TokenPool() + + expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal(true) + expect(tokenPool.add('token')).to.equal(false) + + expect(tokenPool.next().data).to.deep.equal({ + scopes: ['read:packages'], + }) + }) + it('revalidates invalid tokens when they are added again', function () { const tokenPool = new TokenPool() expect(tokenPool.add('token', { scopes: [] })).to.equal(true) From cc21890c9a9c9d0722a8d2623986170939ca3e2e Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 00:35:25 +0800 Subject: [PATCH 13/17] chore(github): defer token scopes index Assisted-by: Codex:gpt-5.5 --- migrations/1780134030603_add-token-scopes.cjs | 2 -- 1 file changed, 2 deletions(-) diff --git a/migrations/1780134030603_add-token-scopes.cjs b/migrations/1780134030603_add-token-scopes.cjs index bc28c1613bb8b..9a820b2133850 100644 --- a/migrations/1780134030603_add-token-scopes.cjs +++ b/migrations/1780134030603_add-token-scopes.cjs @@ -4,10 +4,8 @@ exports.up = pgm => { pgm.addColumn('github_user_tokens', { scopes: { type: 'text[]' }, }) - pgm.createIndex('github_user_tokens', 'scopes', { method: 'gin' }) } exports.down = pgm => { - pgm.dropIndex('github_user_tokens', 'scopes') pgm.dropColumn('github_user_tokens', 'scopes') } From a9fdfc646deb14603e80d71996d1c49dc2c700c8 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 00:35:46 +0800 Subject: [PATCH 14/17] fix(github): trim accepted token scopes Assisted-by: Codex:gpt-5.5 --- services/github/auth/acceptor.js | 5 ++++- services/github/auth/acceptor.spec.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/github/auth/acceptor.js b/services/github/auth/acceptor.js index f8b4e086e0870..f74adaffc549d 100644 --- a/services/github/auth/acceptor.js +++ b/services/github/auth/acceptor.js @@ -65,7 +65,10 @@ function setRoutes({ server, authHelper, onTokenAccepted }) { if (!token) { return end('The GitHub OAuth process did not return a user token.') } - const scopes = scope.split(',').filter(Boolean) + const scopes = scope + .split(',') + .map(scope => scope.trim()) + .filter(Boolean) ask.res.setHeader('Content-Type', 'text/html') end( diff --git a/services/github/auth/acceptor.spec.js b/services/github/auth/acceptor.spec.js index 81bf30b1bd782..03b5c4511b3e6 100644 --- a/services/github/auth/acceptor.spec.js +++ b/services/github/auth/acceptor.spec.js @@ -122,7 +122,7 @@ describe('Github token acceptor', function () { }) it('should pass token scopes from the OAuth response', async function () { - tokenResponse.scope = 'read:packages,read:user' + tokenResponse.scope = 'read:packages, read:user' const form = new FormData() form.append('code', fakeCode) From 50262e27b6f25abde3ab588952afddd4c2936169 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 00:37:11 +0800 Subject: [PATCH 15/17] chore(github): keep token scopes index Assisted-by: Codex:gpt-5.5 --- migrations/1780134030603_add-token-scopes.cjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrations/1780134030603_add-token-scopes.cjs b/migrations/1780134030603_add-token-scopes.cjs index 9a820b2133850..bc28c1613bb8b 100644 --- a/migrations/1780134030603_add-token-scopes.cjs +++ b/migrations/1780134030603_add-token-scopes.cjs @@ -4,8 +4,10 @@ exports.up = pgm => { pgm.addColumn('github_user_tokens', { scopes: { type: 'text[]' }, }) + pgm.createIndex('github_user_tokens', 'scopes', { method: 'gin' }) } exports.down = pgm => { + pgm.dropIndex('github_user_tokens', 'scopes') pgm.dropColumn('github_user_tokens', 'scopes') } From 1e5f65dddbd8c9c9f7b3e8ce3a8d882ebe28d9fb Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 00:41:42 +0800 Subject: [PATCH 16/17] docs(token-pool): document metadata update semantics Assisted-by: Codex:gpt-5.5 --- core/token-pooling/token-pool.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/token-pooling/token-pool.js b/core/token-pooling/token-pool.js index 79de6aec3e633..8c56c4a5d6a1e 100644 --- a/core/token-pooling/token-pool.js +++ b/core/token-pooling/token-pool.js @@ -87,6 +87,8 @@ class Token { /** * Update user-provided data associated with the token. * + * Callers should only invoke this when they have fresh data to store. + * * @param {*} data reserved for future use */ updateData(data) { @@ -232,6 +234,10 @@ class TokenPool { /** * Add a token with user-provided ID and data. * + * Adding an existing token updates its data only when `data` is provided. + * This keeps calls which only revalidate or deduplicate a token from + * clearing previously stored metadata. + * * @param {string} id token string * @param {*} data reserved for future use * @param {number} usesRemaining From a3a38b46728cfb8c182aec14254787770045ecb1 Mon Sep 17 00:00:00 2001 From: DCjanus Date: Sun, 7 Jun 2026 01:20:23 +0800 Subject: [PATCH 17/17] chore(github): store token scopes without index Assisted-by: Codex:gpt-5.5 --- migrations/1780134030603_add-token-scopes.cjs | 2 -- 1 file changed, 2 deletions(-) diff --git a/migrations/1780134030603_add-token-scopes.cjs b/migrations/1780134030603_add-token-scopes.cjs index bc28c1613bb8b..9a820b2133850 100644 --- a/migrations/1780134030603_add-token-scopes.cjs +++ b/migrations/1780134030603_add-token-scopes.cjs @@ -4,10 +4,8 @@ exports.up = pgm => { pgm.addColumn('github_user_tokens', { scopes: { type: 'text[]' }, }) - pgm.createIndex('github_user_tokens', 'scopes', { method: 'gin' }) } exports.down = pgm => { - pgm.dropIndex('github_user_tokens', 'scopes') pgm.dropColumn('github_user_tokens', 'scopes') }