diff --git a/README.md b/README.md index 66138727..b6a9fa02 100644 --- a/README.md +++ b/README.md @@ -476,3 +476,16 @@ trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. + +### Optional incremental session discovery + +Set `AMPLIFIER_SESSION_CATALOG_HINT_DIRECTORY` to an existing canonical absolute, +same-user private POSIX inbox to publish location-only notices after successful +session saves or renames. The CLI emits no transcript/event contents and does not +start a catalog, agent or history watcher. Repeated updates coalesce atomically; +notice failures do not prevent canonical saves. The independent catalog consumer +must be configured explicitly for the same inbox and native roots. Older installed +CLI versions do not gain this behavior from setting the variable alone. + +See [CLI location notices v1](docs/contracts/catalog-location-notices.v1.md) for +bounds, ownership, qualification and the separately unresolved physical-delete seam. diff --git a/amplifier_app_cli/catalog_notices.py b/amplifier_app_cli/catalog_notices.py new file mode 100644 index 00000000..ab6bd2c4 --- /dev/null +++ b/amplifier_app_cli/catalog_notices.py @@ -0,0 +1,81 @@ +"""Opt-in location-only observations after successful CLI native persistence.""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +from pathlib import Path +import stat +import uuid + +logger = logging.getLogger(__name__) +ENVIRONMENT = 'AMPLIFIER_SESSION_CATALOG_HINT_DIRECTORY' +MAX_BYTES = 8192 + + +def announce_saved_session(session_directory: Path) -> bool: + """Best effort only; never affect canonical save success or start an agent. + + The configured private consumer directory must already exist. Do not create + directories, read history, enumerate the spool, or select a native owner. + """ + configured = os.environ.get(ENVIRONMENT) + if not configured: + return False + directory_fd = file_fd = None + temporary = None + try: + if os.name != 'posix': + raise ValueError('Private catalog notices require POSIX') + inbox = Path(configured).expanduser() + if not inbox.is_absolute() or inbox.resolve(strict=True) != inbox: + raise ValueError('Exact existing private notice directory required') + source = Path(session_directory).resolve(strict=True) + if not source.is_dir(): + raise ValueError('Saved session directory required') + raw = json.dumps({'version': 1, 'sessionDirectory': str(source)}, ensure_ascii=False, separators=(',', ':')).encode('utf-8') + if len(raw) > MAX_BYTES: + raise ValueError('Catalog location notice exceeds 8 KiB') + directory_fd = os.open(inbox, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_NONBLOCK) + info = os.fstat(directory_fd) + if not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o077 or info.st_uid != os.getuid(): + raise ValueError('Same-owner private notice directory required') + # Hold the validated directory descriptor through publication so a path + # replacement cannot redirect this write into an unverified directory. + name = hashlib.sha256(str(source).encode('utf-8')).hexdigest() + '.json' + temporary = '.catalog-notice-' + uuid.uuid4().hex + file_fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600, dir_fd=directory_fd) + offset = 0 + while offset < len(raw): + count = os.write(file_fd, raw[offset:]) + if count <= 0: + raise OSError('Catalog notice write did not advance') + offset += count + os.fsync(file_fd) + os.close(file_fd); file_fd = None + os.replace(temporary, name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) + temporary = None + os.fsync(directory_fd) + return True + except Exception as exc: + # No configured paths, metadata, exception payloads or credentials in + # diagnostics. A notice failure cannot roll back native persistence. + logger.debug('Catalog notice not confirmed (%s)', type(exc).__name__) + return False + finally: + if file_fd is not None: + try: + os.close(file_fd) + except OSError: + pass + if temporary is not None and directory_fd is not None: + try: + os.unlink(temporary, dir_fd=directory_fd) + except OSError: + pass + if directory_fd is not None: + try: + os.close(directory_fd) + except OSError: + pass diff --git a/amplifier_app_cli/session_store.py b/amplifier_app_cli/session_store.py index 8c219f82..67b35ab5 100644 --- a/amplifier_app_cli/session_store.py +++ b/amplifier_app_cli/session_store.py @@ -23,6 +23,7 @@ from amplifier_foundation.session.history import SessionHistoryStore from amplifier_foundation.session.metadata import SessionMetadataStore +from amplifier_app_cli.catalog_notices import announce_saved_session from amplifier_app_cli.project_utils import get_project_slug from amplifier_foundation.paths.resolution import get_amplifier_home @@ -129,6 +130,7 @@ def save(self, session_id: str, transcript: list, metadata: dict) -> None: transcript, redact_secrets(metadata), sanitizer=sanitize_message, merge_metadata=True ) + announce_saved_session(session_dir) logger.debug(f"Session {session_id} saved successfully") def reserve_session(self, session_id: str) -> Path: @@ -159,6 +161,7 @@ def save_new(self, session_id: str, transcript: list, metadata: dict) -> None: # write cannot delete another session's data. shutil.rmtree(session_dir, ignore_errors=True) raise + announce_saved_session(session_dir) logger.debug(f"New session {session_id} saved successfully") def _save_transcript(self, session_dir: Path, transcript: list) -> None: @@ -171,10 +174,12 @@ def _save_transcript(self, session_dir: Path, transcript: list) -> None: SessionHistoryStore(session_dir).save_messages( transcript, sanitizer=sanitize_message ) + announce_saved_session(session_dir) def _save_metadata(self, session_dir: Path, metadata: dict) -> None: """Save native metadata with the CLI's existing credential redaction.""" SessionHistoryStore(session_dir).save_metadata(redact_secrets(metadata), merge_metadata=True) + announce_saved_session(session_dir) def load(self, session_id: str) -> tuple[list, dict]: """Load session state with corruption recovery. @@ -255,13 +260,16 @@ def update_metadata(self, session_id: str, updates: dict) -> dict: metadata = SessionMetadataStore(session_dir).update(redact_secrets(updates)) + announce_saved_session(session_dir) logger.debug(f"Session {session_id} metadata updated: {list(updates.keys())}") return metadata def rename(self, session_id: str, name: str) -> dict: """Rename through Foundation without replacing transcript/runtime state.""" self.get_metadata(session_id) # Keep strict identity/existence validation. - return SessionMetadataStore(self.base_dir / session_id).set_name(name) + metadata = SessionMetadataStore(self.base_dir / session_id).set_name(name) + announce_saved_session(self.base_dir / session_id) + return metadata def get_metadata(self, session_id: str) -> dict: """Get session metadata without loading transcript. diff --git a/docs/contracts/catalog-location-notices.v1.md b/docs/contracts/catalog-location-notices.v1.md new file mode 100644 index 00000000..d08cac44 --- /dev/null +++ b/docs/contracts/catalog-location-notices.v1.md @@ -0,0 +1,17 @@ +# CLI location notices, v1 + +The CLI owns optional announcements after successful native persistence. Foundation remains the owner of canonical transcript/metadata writes and shared writer locks; the independent catalog owns derived discovery. This uses the existing catalog location-only inbox contract, not a new history store or execution receipt. + +`AMPLIFIER_SESSION_CATALOG_HINT_DIRECTORY` opts into an existing, canonical absolute, same-user private POSIX directory (mode 0700). An unset variable performs no notice I/O. Invalid, missing, nonprivate or symlinked destinations never prevent a successful native save. The CLI does not create the inbox, inspect its contents, start a catalog or agent, activate providers, or watch historical directories. + +After `SessionStore.save`, `save_new`, `_save_transcript`, `_save_metadata`, `update_metadata` or `rename` succeeds, the CLI publishes only `{"version":1,"sessionDirectory":"/canonical/absolute/native/session/directory"}`. No title, transcript, event, credential, actor or visibility fields are included. Existing shared-root checkpoint and actual `/rename` paths delegate through these covered wrappers. Failed canonical writes emit no notice. + +The compact UTF-8 body is at most 8 KiB. Its filename is SHA-256 of the canonical UTF-8 source path plus `.json`. A same-directory unique mode-0600 temporary file is fully written and fsynced, atomically replaced into that filename, then the held inbox descriptor is fsynced. Publication uses an already validated directory descriptor, preventing a later pathname replacement from redirecting writes. Repeated saves coalesce; a consumer's existing `.processing-...` claim is preserved. Temporary cleanup and notice failures never roll back native persistence. Notice delivery is best effort and diagnostic output contains only exception class, not configured paths or source payloads. + +The consumer validates configured native roots and rereads current metadata. A notice is neither proof of native save outcome nor deletion authority. Physical CLI deletion is outside this producer patch. It must not manufacture another owner's `lifecycle.json`, mark nativeDeleted, change host productHidden, or treat missing metadata as an authoritative tombstone. Existing configured native-owner lifecycle markers remain separately identity checked by the catalog. Default/background metadata reconciliation can repair missed notices, but synthetic consumer acceptance does not establish adoption by older installed CLI versions. + +## Reviewed implementations and checks + +The local ecosystem catalog `amplifier/docs/MODULES.md` identifies CLI, Foundation and independent application ownership. The CLI baseline is `5aaafb478d02cee8c396967b954299a08e8a1efd`. All CLI-owned `SessionHistoryStore` and `SessionMetadataStore` write call sites are centralized in the wrappers above; `SharedRootSession.checkpoint` calls native_store.save. Native ACP and Foundation save implementations remain unchanged. No portable runtime module or bundle asset is added. + +Tests use the installed Foundation dependency and cover all write variants, actual CLI rename, source-byte preservation, disabled/malformed/private-directory gates, bounded exact payloads, atomic/fsynced publication, consumer-claim preservation, opened-directory replacement, canonical save failure and notice failure. `scripts/qualify_catalog_notices.py` additionally requires an independently installed catalog and exercises actual installed CLI save/rename, coalesced updates and configured native-owner lifecycle proof on isolated physical fixtures. That fixture does not qualify real CLI deletion, provider/account work, browser/device rendering, or live installation. diff --git a/scripts/qualify_catalog_notices.py b/scripts/qualify_catalog_notices.py new file mode 100644 index 00000000..b4b81ecb --- /dev/null +++ b/scripts/qualify_catalog_notices.py @@ -0,0 +1,74 @@ +"""Real installed CLI producer plus separately installed catalog; fixture-only.""" +import argparse +import asyncio +import hashlib +import json +import os +from pathlib import Path +import stat +import tempfile +from types import SimpleNamespace + +from amplifier_app_cli.catalog_notices import ENVIRONMENT +from amplifier_app_cli.main import CommandProcessor +from amplifier_app_cli.session_store import SessionStore +from amplifier_session_catalog import Catalog +from amplifier_session_catalog.discovery import Discovery +from amplifier_session_catalog.hints import HintInbox + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--output',required=True);args=parser.parse_args() + report={'scope':'Installed CLI save/rename producer and independently installed catalog on isolated physical sources; no agents/provider calls/live mutation', + 'cliPackage':__import__('amplifier_app_cli').__file__,'catalogPackage':__import__('amplifier_session_catalog').__file__} + with tempfile.TemporaryDirectory(prefix='cli-catalog-producer-') as temporary: + root=Path(temporary).resolve();workspace=root/'workspace';workspace.mkdir();home=root/'native';inbox=root/'hints';inbox.mkdir(mode=0o700) + previous={key:os.environ.get(key) for key in (ENVIRONMENT,'AMPLIFIER_HOME')};cwd=Path.cwd() + os.environ[ENVIRONMENT]=str(inbox);os.environ['AMPLIFIER_HOME']=str(home);os.chdir(workspace) + try: + store=SessionStore();catalog=Catalog(root/'index.sqlite');app=root/'configured-native-owner';discovery=Discovery(catalog,[home],app_homes=[app]);consumer=HintInbox(inbox,discovery) + store.save_new('selected',[{'role':'user','content':'Canonical selected text'}],{'working_dir':str(workspace),'name':'Initial'}) + directory=store.base_dir/'selected';transcript=(directory/'transcript.jsonl').read_bytes();history_digest=hashlib.sha256(transcript).hexdigest() + assert consumer.drain(limit=1)['processed']==1;consumer.close() + selected=catalog.list(connectionId='producer')['items'][0];assert selected['nativeSessionId']=='selected' + # Actual /rename implementation, not a substitute metadata mutation. + processor=CommandProcessor.__new__(CommandProcessor);processor.session=SimpleNamespace(coordinator=SimpleNamespace(session_id='selected')) + assert 'CLI name' in asyncio.run(processor._rename_session('CLI name')) + for number in range(5):store.update_metadata('selected',{'name':'CLI name','description':f'Coalesced {number}'}) + notices=list(inbox.iterdir());assert len(notices)==1 and notices[0].stat().st_size<=8192 and stat.S_IMODE(notices[0].stat().st_mode)==0o600 + value=json.loads(notices[0].read_text());assert value=={'version':1,'sessionDirectory':str(directory.resolve())} + opens=[];original=os.open + def guarded(path,*a,**k): + path=Path(path);assert path.name not in {'events.jsonl','transcript.jsonl','transcript.jsonl.backup'} + if path.name in {'metadata.json','metadata.json.backup','lifecycle.json'}:opens.append(str(path)) + return original(path,*a,**k) + os.open=guarded + try:assert consumer.drain(limit=1)['processed']==1 + finally:os.open=original;consumer.close() + assert len(opens)==1 and catalog.get(selected['uri'])['title']=='CLI name' and catalog.get(selected['uri'])['description']=='Coalesced 4' + assert (directory/'transcript.jsonl').read_bytes()==transcript + # Existing configured native owner supplies lifecycle authority; + # the CLI notice carries only location and does not forge tombstones. + marker=app/'sessions'/'selected'/'lifecycle.json';marker.parent.mkdir(parents=True) + marker.write_text(json.dumps({'sessionId':'selected','historyCwd':str(workspace),'deleted':True,'commandId':'native-owner-delete'})) + store.update_metadata('selected',{'description':'Owner deletion notice trigger'}) + assert consumer.drain(limit=1)['processed']==1;consumer.close();assert catalog.get(selected['uri'])['nativeDeleted'] and not catalog.list(connectionId='deleted')['items'] + marker.write_text(json.dumps({'sessionId':'selected','historyCwd':str(workspace),'deleted':False,'commandId':'native-owner-restore'})) + store.update_metadata('selected',{'description':'Owner restoration notice trigger'}) + assert consumer.drain(limit=1)['processed']==1;consumer.close();assert not catalog.get(selected['uri'])['nativeDeleted'] + # A producer failure cannot veto the CLI canonical save. + os.environ[ENVIRONMENT]=str(root/'uncreated-inbox') + store.rename('selected','Canonical success despite missing inbox');assert store.get_metadata('selected')['name']=='Canonical success despite missing inbox' and not (root/'uncreated-inbox').exists() + assert (directory/'transcript.jsonl').read_bytes()==transcript + report.update(newSession='discovered from actual CLI save',rename='actual CLI /rename action discovered',coalescedUpdates=5,coalescedNoticeFiles=1,renameConsumerMetadataReads=1,transcriptSha256=history_digest,canonicalHistoryPreserved=True, + nativeOwnerTombstone='existing configured owner proof consumed; CLI emitted no lifecycle authority',nativeOwnerRestore='consumed',failedOptIn='canonical save succeeded; missing inbox not created',nativeTranscriptEventReadsByNoticeConsumer=0,agentsStarted=0,providerCalls=0) + report['limits']=['Physical legacy CLI delete has no authoritative tombstone integration in this patch','Existing tombstone fixture is supplied by configured native owner, not invented by CLI','No watcher/background historical directory tracking','No browser/device/real-account/live deployment acceptance'] + finally: + os.chdir(cwd) + for key,value in previous.items(): + if value is None:os.environ.pop(key,None) + else:os.environ[key]=value + Path(args.output).write_text(json.dumps(report,indent=2)+'\n');print(json.dumps(report)) + + +if __name__=='__main__':main() diff --git a/tests/test_catalog_notices.py b/tests/test_catalog_notices.py new file mode 100644 index 00000000..1343b9e0 --- /dev/null +++ b/tests/test_catalog_notices.py @@ -0,0 +1,139 @@ +"""CLI-owned producer tests use the installed Foundation dependency.""" +import hashlib +import json +import os +from pathlib import Path +import stat + +import pytest + +from amplifier_app_cli.catalog_notices import ENVIRONMENT, announce_saved_session +from amplifier_app_cli.session_store import SessionStore + +pytestmark = pytest.mark.skipif(os.name != 'posix', reason='Private catalog notice v1 is POSIX-only') + + +def configured(tmp_path,monkeypatch): + inbox=tmp_path/'hints';inbox.mkdir(mode=0o700) + monkeypatch.setenv(ENVIRONMENT,str(inbox)) + store=SessionStore(tmp_path/'native'/'sessions') + return store,inbox + + +def payload(inbox,directory): + name=hashlib.sha256(str(directory.resolve()).encode()).hexdigest()+'.json';path=inbox/name + assert stat.S_IMODE(path.stat().st_mode)==0o600 + assert path.stat().st_size<=8192 + value=json.loads(path.read_text());assert value=={'version':1,'sessionDirectory':str(directory.resolve())} + assert sorted(p.name for p in inbox.iterdir())==[name] + return path + + +def test_successful_native_write_variants_coalesce_exact_location_only_notices(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch);directory=store.base_dir/'one' + store.save_new('one',[{'role':'user','content':'private original text'}],{'working_dir':str(tmp_path),'name':'Original','access_token':'hidden'}) + path=payload(inbox,directory);before=(directory/'transcript.jsonl').read_bytes() + store.rename('one','Renamed');payload(inbox,directory) + store.update_metadata('one',{'description':'Private description'});payload(inbox,directory) + store._save_metadata(directory,{'working_dir':str(tmp_path),'name':'Merged update'});payload(inbox,directory) + assert (directory/'transcript.jsonl').read_bytes()==before + store._save_transcript(directory,[{'role':'user','content':'Private changed text'}]);payload(inbox,directory) + store.save('one',[{'role':'user','content':'Final private text'}],{'working_dir':str(tmp_path),'name':'Final'});payload(inbox,directory) + assert 'Final private text' not in path.read_text() and 'access_token' not in path.read_text() + + +def test_opt_out_does_not_create_notice_directory(tmp_path,monkeypatch): + monkeypatch.delenv(ENVIRONMENT,raising=False) + store=SessionStore(tmp_path/'native');store.save_new('one',[],{'name':'Saved'}) + assert not (tmp_path/'hints').exists() and announce_saved_session(store.base_dir/'one') is False + + +@pytest.mark.parametrize('kind',['missing','relative','public','symlink','foreign-directory-entry']) +def test_bad_opt_in_cannot_fail_canonical_save_or_change_foreign_files(tmp_path,monkeypatch,kind): + store,inbox=configured(tmp_path,monkeypatch) + if kind=='missing':monkeypatch.setenv(ENVIRONMENT,str(tmp_path/'absent')) + elif kind=='relative':monkeypatch.setenv(ENVIRONMENT,'relative-inbox') + elif kind=='public':inbox.chmod(0o755) + elif kind=='symlink': + link=tmp_path/'alias';link.symlink_to(inbox);monkeypatch.setenv(ENVIRONMENT,str(link)) + else: + name=hashlib.sha256(str((store.base_dir/'one').resolve()).encode()).hexdigest()+'.json' + foreign=inbox/name;foreign.mkdir();(foreign/'preserved').write_text('foreign contents') + store.save_new('one',[{'role':'user','content':'canonical'}],{'name':'Saved'}) + assert store.get_metadata('one')['name']=='Saved' + assert not (tmp_path/'absent').exists() and not any(p.name.startswith('.catalog-notice-') for p in inbox.iterdir()) + if kind=='foreign-directory-entry':assert (foreign/'preserved').read_text()=='foreign contents' + + +def test_notice_after_canonical_failure_is_never_emitted(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch) + from amplifier_foundation.session.history import SessionHistoryStore + def failed(*args,**kwargs):raise OSError('fixture persistence failure') + monkeypatch.setattr(SessionHistoryStore,'save',failed) + with pytest.raises(OSError):store.save('one',[],{}) + assert not list(inbox.iterdir()) + with pytest.raises(OSError):store.save_new('two',[],{}) + assert not list(inbox.iterdir()) and not (store.base_dir/'two').exists() + + +def test_emitter_reads_no_canonical_bodies_or_spool_listing(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch);directory=store.base_dir/'one';directory.mkdir() + def forbidden(*args,**kwargs):raise AssertionError('Emitter read/listed source bodies') + with monkeypatch.context() as m: + m.setattr(Path,'open',forbidden);m.setattr(os,'scandir',forbidden) + assert announce_saved_session(directory) + payload(inbox,directory) + + +def test_atomic_publication_is_fsynced_and_replaces_claim_without_deleting_it(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch);directory=store.base_dir/'one';directory.mkdir() + assert announce_saved_session(directory) + first=payload(inbox,directory);claim=first.with_name(first.stem+'.processing-'+'a'*32);first.rename(claim) + original=os.fsync;synced=[] + def observed(fd):synced.append(stat.S_ISDIR(os.fstat(fd).st_mode));return original(fd) + monkeypatch.setattr(os,'fsync',observed) + assert announce_saved_session(directory) and claim.exists() and first.exists() and synced==[False,True] + assert json.loads(first.read_text())==json.loads(claim.read_text()) + + +def test_opened_directory_identity_prevents_redirect_after_path_swap(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch);directory=store.base_dir/'one';directory.mkdir() + other=tmp_path/'other';other.mkdir(mode=0o755);moved=tmp_path/'owned-moved' + original=os.open;swapped=False + def opened(path,flags,*args,**kwargs): + nonlocal swapped + fd=original(path,flags,*args,**kwargs) + if Path(path)==inbox and not swapped: + swapped=True;inbox.rename(moved);inbox.symlink_to(other) + return fd + monkeypatch.setattr(os,'open',opened) + assert announce_saved_session(directory) + assert not list(other.iterdir()) and len(list(moved.iterdir()))==1 + + +def test_fsync_failure_and_close_failure_preserve_canonical_save(tmp_path,monkeypatch): + store,inbox=configured(tmp_path,monkeypatch) + original=os.fsync + def failed(fd): + if os.fstat(fd).st_ino==inbox.stat().st_ino:raise OSError('private fixture durability failure') + return original(fd) + # Only announcement directory fsync fails; Foundation canonical writes finish. + monkeypatch.setattr(os,'fsync',failed) + store.save('one',[{'role':'user','content':'preserved'}],{'name':'Preserved'}) + assert store.get_metadata('one')['name']=='Preserved' + + +@pytest.mark.asyncio +async def test_actual_cli_rename_action_emits_after_native_metadata_only_change(tmp_path,monkeypatch): + from types import SimpleNamespace + from amplifier_app_cli.main import CommandProcessor + workspace=tmp_path/'workspace';workspace.mkdir();monkeypatch.chdir(workspace) + monkeypatch.setenv('AMPLIFIER_HOME',str(tmp_path/'native-home')) + inbox=tmp_path/'hints';inbox.mkdir(mode=0o700);monkeypatch.setenv(ENVIRONMENT,str(inbox)) + store=SessionStore();store.save_new('selected',[{'role':'user','content':'Preserved transcript'}],{'working_dir':str(workspace),'name':'Before'}) + directory=store.base_dir/'selected';transcript=(directory/'transcript.jsonl').read_bytes() + processor=CommandProcessor.__new__(CommandProcessor);processor.session=SimpleNamespace(coordinator=SimpleNamespace(session_id='selected')) + result=await processor._rename_session('Actual CLI rename') + assert 'Actual CLI rename' in result and store.get_metadata('selected')['name']=='Actual CLI rename' + assert (directory/'transcript.jsonl').read_bytes()==transcript + payload(inbox,directory)