Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 62 additions & 15 deletions src/gmail-ai-classifier/src/github-sync.js
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ function appendMarkdownEntryToGitHubRepo(filePath, entryMd, commitMessage) {
commitMessage,
githubToken
)
if (result === true || result === 'IDEMPOTENT_SKIP') {
if (result === true) {
Comment thread
don-petry marked this conversation as resolved.
Resolved
return true
}
console.log(
Expand Down Expand Up @@ -137,12 +137,12 @@ function executeGitHubCommit(filePath, entryMd, commitMessage, githubToken) {
rawContent = decodeBase64Content(fileData.content)

// Idempotency Check: Skip if entry already present
if (rawContent.indexOf(entryMd.trim()) !== -1) {
if (rawContent.includes(entryMd.trim())) {
console.log(
'[gitHubSync] Idempotent Skip: Entry already exists in',
filePath
)
return 'IDEMPOTENT_SKIP'
return true
}
} else {
console.error(
Expand Down Expand Up @@ -206,7 +206,7 @@ function executeGitHubCommit(filePath, entryMd, commitMessage, githubToken) {
function extractTopicTitleFromPath(filePath) {
var parts = filePath.split('/')
var topic = parts.length > 1 ? parts[parts.length - 2] : parts[0]
return topic.replace(/-/g, ' ').replace(/\b\w/g, function (l) {
return topic.replaceAll('-', ' ').replace(/\b\w/g, function (l) {
return l.toUpperCase()
})
}
Expand Down Expand Up @@ -288,6 +288,57 @@ function fetchRulesFromGitHub(filePath, githubToken) {
return null
}

/**
* Compares two rules objects for content equality, ignoring the internal _sha field.
*/
function rulesContentEquals_(a, b) {
var aCopy = { ...a }
delete aCopy._sha
var bCopy = { ...b }
delete bCopy._sha
return JSON.stringify(aCopy) === JSON.stringify(bCopy)
}

/**
* Decides whether it is safe to overwrite the freshly-fetched remote rules with the
* local copy after a 409 conflict.
*
* Timestamps are parsed to epoch millis (normalizing timezone offsets, e.g. a
* `+00:00` vs `Z` suffix, to the same instant) rather than compared as strings.
* When either timestamp is missing/invalid or the two are equal, we cannot rank
* them by time, so we fall back to a content comparison: if the remote already
* differs from our local copy we abort rather than clobber a possibly-newer remote.
*
* @param {Object} remoteRules - Freshly fetched remote rules (may include _sha)
* @param {Object} localRules - Local rules about to be committed
* @param {string} localUpdatedAt - Local rules' updatedAt timestamp
* @returns {boolean} true if overwriting the remote is safe, false to abort
*/
function isSafeToOverwriteRemote_(remoteRules, localRules, localUpdatedAt) {
var remoteMs = new Date(remoteRules.updatedAt).getTime()
var localMs = new Date(localUpdatedAt).getTime()
var timestampsComparable =
!Number.isNaN(remoteMs) && !Number.isNaN(localMs) && remoteMs !== localMs

if (timestampsComparable) {
if (remoteMs > localMs) {
console.error('[gitHubSync] Remote copy is newer; aborting commit')
return false
}
return true
}

// Timestamps are missing, invalid, or represent the same instant: only overwrite
// when the remote content matches what we already have.
if (!rulesContentEquals_(remoteRules, localRules)) {
console.error(
'[gitHubSync] Timestamps inconclusive and remote content differs; aborting commit'
)
return false
}
return true
}

/**
* 2-Way Sync Engine: Commits rules object back to self-private via GitHub REST API.
* @param {Object} rulesObj - Rules to commit (mutated in-place: _sha deleted, updatedAt set)
Expand Down Expand Up @@ -355,14 +406,12 @@ function commitRulesToGitHub(rulesObj, commitMessage, githubToken, knownSha) {
var status = putRes.getResponseCode()
if (status === 200 || status === 201) return true
if (status === 409) {
// SHA conflict: refresh and check if remote is newer
// SHA conflict: refresh and decide whether it is safe to overwrite remote.
var refreshed = fetchRulesFromGitHub(targetPath, token)
if (
refreshed &&
refreshed.updatedAt &&
refreshed.updatedAt > localUpdatedAt
!isSafeToOverwriteRemote_(refreshed, rulesObj, localUpdatedAt)
) {
console.error('[gitHubSync] Remote copy is newer; aborting commit')
return false
}
var freshSha = refreshed ? refreshed._sha : null
Expand Down Expand Up @@ -422,8 +471,8 @@ function syncTwoWayRules() {

var remoteTs = new Date(remoteRules.updatedAt).getTime()
var localTs = new Date(localRules.updatedAt).getTime()
var remoteDate = isNaN(remoteTs) ? 0 : remoteTs
var localDate = isNaN(localTs) ? 0 : localTs
var remoteDate = Number.isNaN(remoteTs) ? 0 : remoteTs
var localDate = Number.isNaN(localTs) ? 0 : localTs

if (remoteDate > localDate) {
// GitHub rules are newer -> Update GAS Script Property
Expand All @@ -448,11 +497,7 @@ function syncTwoWayRules() {
}

// Equal timestamps: use content comparison as tie-breaker (exclude internal _sha field)
var remoteForCompare = JSON.parse(JSON.stringify(remoteRules))
delete remoteForCompare._sha
var localForCompare = JSON.parse(JSON.stringify(localRules))
delete localForCompare._sha
if (JSON.stringify(remoteForCompare) !== JSON.stringify(localForCompare)) {
if (!rulesContentEquals_(remoteRules, localRules)) {
props.setProperty('CLASSIFICATION_RULES_JSON', JSON.stringify(remoteRules))
console.log(
'[gitHubSync] Equal timestamps but content differed — pulled remote as canonical.'
Expand All @@ -473,6 +518,8 @@ if (typeof module !== 'undefined' && module.exports) {
extractTopicTitleFromPath: extractTopicTitleFromPath,
insertEntryIntoLogSection: insertEntryIntoLogSection,
fetchRulesFromGitHub: fetchRulesFromGitHub,
rulesContentEquals_: rulesContentEquals_,
isSafeToOverwriteRemote_: isSafeToOverwriteRemote_,
commitRulesToGitHub: commitRulesToGitHub,
syncTwoWayRules: syncTwoWayRules,
getGitHubApiUrl_: getGitHubApiUrl_,
Expand Down
175 changes: 166 additions & 9 deletions src/gmail-ai-classifier/tests/gitHubSync.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ const {
fetchRulesFromGitHub,
commitRulesToGitHub,
syncTwoWayRules,
rulesContentEquals_,
isSafeToOverwriteRemote_,
} = require('../src/github-sync.js')

describe('gitHubSync Module', () => {
Expand Down Expand Up @@ -95,24 +97,25 @@ describe('gitHubSync Module', () => {
expect(res).toBe(true)
})

test('handles 200 with idempotent skip', () => {
test('handles 200 with idempotent skip (returns boolean true)', () => {
const existingContent = '- entry'
const base64 = Buffer.from(existingContent).toString('base64')
global.UrlFetchApp = {
fetch: jest.fn(() => ({
getResponseCode: () => 200,
getContentText: () =>
JSON.stringify({ content: base64, sha: 'sha-1' }),
})),
}
const putSpy = jest.fn(() => ({
getResponseCode: () => 200,
getContentText: () => JSON.stringify({ content: base64, sha: 'sha-1' }),
}))
global.UrlFetchApp = { fetch: putSpy }

const res = executeGitHubCommit(
'01_Household/test/index.md',
'- entry',
'commit msg',
'pat-123'
)
expect(res).toBe('IDEMPOTENT_SKIP')
// Idempotent skip is a success: returns boolean true (unified return type),
// and performs no PUT (only the initial GET).
expect(res).toBe(true)
expect(putSpy).toHaveBeenCalledTimes(1)
})

test('handles 500 error fetching file', () => {
Expand Down Expand Up @@ -932,5 +935,159 @@ describe('gitHubSync Module', () => {
expect(result).toBe(false)
expect(global.UrlFetchApp.fetch).toHaveBeenCalledTimes(2)
})

test('commitRulesToGitHub aborts on 409 when remote is newer despite an earlier-sorting timezone offset', () => {
// Same instant would be 15:00Z (newer than the local 14:00Z), but the string
// "09:00:00-06:00" sorts *before* "14:00:00Z", which a naive string compare
// would treat as older and wrongly overwrite. Numeric comparison must abort.
const remoteRules = { updatedAt: '2026-08-08T09:00:00-06:00' }
const base64 = Buffer.from(JSON.stringify(remoteRules)).toString('base64')
global.UrlFetchApp = {
fetch: jest
.fn()
.mockImplementationOnce(() => ({
getResponseCode: () => 409,
getContentText: () => 'Conflict',
}))
.mockImplementationOnce(() => ({
getResponseCode: () => 200,
getContentText: () =>
JSON.stringify({ content: base64, sha: 'fresh-sha' }),
})),
}
const result = commitRulesToGitHub({}, 'msg', 'token', 'stale-sha')
expect(result).toBe(false)
expect(global.UrlFetchApp.fetch).toHaveBeenCalledTimes(2)
})

test('commitRulesToGitHub aborts on 409 when timestamps are the same instant but content differs', () => {
// Local updatedAt is mocked to 2026-08-08T14:00:00Z; remote uses an equivalent
// instant with an explicit +00:00 offset but different content.
const remoteRules = {
version: '9.9.9',
updatedAt: '2026-08-08T14:00:00+00:00',
}
const base64 = Buffer.from(JSON.stringify(remoteRules)).toString('base64')
global.UrlFetchApp = {
fetch: jest
.fn()
.mockImplementationOnce(() => ({
getResponseCode: () => 409,
getContentText: () => 'Conflict',
}))
.mockImplementationOnce(() => ({
getResponseCode: () => 200,
getContentText: () =>
JSON.stringify({ content: base64, sha: 'fresh-sha' }),
})),
}
const result = commitRulesToGitHub(
{ version: '1.0.0' },
'msg',
'token',
'stale-sha'
)
expect(result).toBe(false)
expect(global.UrlFetchApp.fetch).toHaveBeenCalledTimes(2)
})

test('commitRulesToGitHub retries on 409 when timestamps are equal and content is identical', () => {
// Remote and local share the same updatedAt and content, so overwriting is safe.
const remoteRules = {
version: '1.0.0',
updatedAt: '2026-08-08T14:00:00Z',
}
const base64 = Buffer.from(JSON.stringify(remoteRules)).toString('base64')
global.UrlFetchApp = {
fetch: jest
.fn()
.mockImplementationOnce(() => ({
getResponseCode: () => 409,
getContentText: () => 'Conflict',
}))
.mockImplementationOnce(() => ({
getResponseCode: () => 200,
getContentText: () =>
JSON.stringify({ content: base64, sha: 'fresh-sha' }),
}))
.mockImplementationOnce(() => ({
getResponseCode: () => 201,
getContentText: () => '{}',
})),
}
const result = commitRulesToGitHub(
{ version: '1.0.0' },
'msg',
'token',
'stale-sha'
)
expect(result).toBe(true)
expect(global.UrlFetchApp.fetch).toHaveBeenCalledTimes(3)
})

test('commitRulesToGitHub aborts on 409 when a timestamp is invalid and content differs', () => {
const remoteRules = { version: '2.0.0', updatedAt: 'not-a-real-date' }
const base64 = Buffer.from(JSON.stringify(remoteRules)).toString('base64')
global.UrlFetchApp = {
fetch: jest
.fn()
.mockImplementationOnce(() => ({
getResponseCode: () => 409,
getContentText: () => 'Conflict',
}))
.mockImplementationOnce(() => ({
getResponseCode: () => 200,
getContentText: () =>
JSON.stringify({ content: base64, sha: 'fresh-sha' }),
})),
}
const result = commitRulesToGitHub(
{ version: '1.0.0' },
'msg',
'token',
'stale-sha'
)
expect(result).toBe(false)
expect(global.UrlFetchApp.fetch).toHaveBeenCalledTimes(2)
})

test('rulesContentEquals_ ignores the internal _sha field', () => {
expect(
rulesContentEquals_(
{ version: '1.0.0', _sha: 'abc' },
{ version: '1.0.0', _sha: 'xyz' }
)
).toBe(true)
expect(
rulesContentEquals_({ version: '1.0.0' }, { version: '2.0.0' })
).toBe(false)
})

test('isSafeToOverwriteRemote_ handles valid, equal, and invalid timestamps', () => {
// Remote strictly newer by instant -> not safe.
expect(
isSafeToOverwriteRemote_(
{ updatedAt: '2026-08-08T15:00:00Z', version: '1.0.0' },
{ updatedAt: '2026-08-08T14:00:00Z', version: '1.0.0' },
'2026-08-08T14:00:00Z'
)
).toBe(false)
// Remote older by instant -> safe regardless of content.
expect(
isSafeToOverwriteRemote_(
{ updatedAt: '2026-08-08T13:00:00Z', version: '9.9.9' },
{ updatedAt: '2026-08-08T14:00:00Z', version: '1.0.0' },
'2026-08-08T14:00:00Z'
)
).toBe(true)
// Equal instant, identical content -> safe.
expect(
isSafeToOverwriteRemote_(
{ updatedAt: '2026-08-08T14:00:00Z', version: '1.0.0' },
{ updatedAt: '2026-08-08T14:00:00Z', version: '1.0.0' },
'2026-08-08T14:00:00Z'
)
).toBe(true)
})
})
})
Loading