feat: drop-in observability kit with audit comparison and behavioral signals

actions/setup/js/generate_observability_summary.cjs

@@ -0,0 +1,133 @@
+// @ts-check
+/// <reference types="@actions/github-script" />
+
+const fs = require("fs");
+
+const AW_INFO_PATH = "/tmp/gh-aw/aw_info.json";
+const AGENT_OUTPUT_PATH = "/tmp/gh-aw/agent_output.json";
+const gatewayEventPaths = ["/tmp/gh-aw/mcp-logs/gateway.jsonl", "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"];
+
+function readJSONIfExists(path) {
+  if (!fs.existsSync(path)) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(path, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function countBlockedRequests() {
+  for (const path of gatewayEventPaths) {
+    if (!fs.existsSync(path)) {
+      continue;
+    }
+
+    const content = fs.readFileSync(path, "utf8");
+    return content
+      .split("\n")
+      .map(line => line.trim())
+      .filter(Boolean)
+      .reduce((count, line) => {
+        try {
+          const entry = JSON.parse(line);
+          return entry && entry.type === "DIFC_FILTERED" ? count + 1 : count;
+        } catch {
+          return count;
+        }
+      }, 0);
+  }
+
+  return 0;
+}
+
+function uniqueCreatedItemTypes(items) {
+  const types = new Set();
+
+  for (const item of items) {
+    if (item && typeof item.type === "string" && item.type.trim() !== "") {
+      types.add(item.type);
+    }
+  }
+
+  return [...types].sort();
+}
+
+function collectObservabilityData() {
+  const awInfo = readJSONIfExists(AW_INFO_PATH) || {};
+  const agentOutput = readJSONIfExists(AGENT_OUTPUT_PATH) || { items: [], errors: [] };
+  const items = Array.isArray(agentOutput.items) ? agentOutput.items : [];
+  const errors = Array.isArray(agentOutput.errors) ? agentOutput.errors : [];
+  const traceId = awInfo.context && typeof awInfo.context.workflow_call_id === "string" ? awInfo.context.workflow_call_id : "";
+
+  return {
+    workflowName: awInfo.workflow_name || "",
+    engineId: awInfo.engine_id || "",
+    traceId,
+    staged: awInfo.staged === true,
+    firewallEnabled: awInfo.firewall_enabled === true,
+    createdItemCount: items.length,
+    createdItemTypes: uniqueCreatedItemTypes(items),
+    outputErrorCount: errors.length,
+    blockedRequests: countBlockedRequests(),
+  };
+}
+
+function buildObservabilitySummary(data) {
+  const posture = data.createdItemCount > 0 ? "write-capable" : "read-only";
+  const lines = [];
+
+  lines.push("<details>");
+  lines.push("<summary><b>Observability</b></summary>");
+  lines.push("");
+
+  if (data.workflowName) {
+    lines.push(`- **workflow**: ${data.workflowName}`);
+  }
+  if (data.engineId) {
+    lines.push(`- **engine**: ${data.engineId}`);
+  }
+  if (data.traceId) {
+    lines.push(`- **trace id**: ${data.traceId}`);
+  }
+
+  lines.push(`- **posture**: ${posture}`);
+  lines.push(`- **created items**: ${data.createdItemCount}`);
+  lines.push(`- **blocked requests**: ${data.blockedRequests}`);
+  lines.push(`- **agent output errors**: ${data.outputErrorCount}`);
+  lines.push(`- **firewall enabled**: ${data.firewallEnabled}`);
+  lines.push(`- **staged**: ${data.staged}`);
+
+  if (data.createdItemTypes.length > 0) {
+    lines.push("- **item types**:");
+    for (const itemType of data.createdItemTypes) {
+      lines.push(`  - ${itemType}`);
+    }
+  }
+
+  lines.push("");
+  lines.push("</details>");
+
+  return lines.join("\n") + "\n";
+}
+
+async function main(core) {
+  const mode = process.env.GH_AW_OBSERVABILITY_JOB_SUMMARY || "";
+  if (mode !== "on") {
+    core.info(`Skipping observability summary: mode=${mode || "unset"}`);
+    return;
+  }
+
+  const data = collectObservabilityData();
+  const markdown = buildObservabilitySummary(data);
+  await core.summary.addRaw(markdown).write();
+  core.info("Generated observability summary in step summary");
+}
+
+module.exports = {
+  buildObservabilitySummary,
+  collectObservabilityData,
+  main,
+};

actions/setup/js/generate_observability_summary.test.cjs

@@ -0,0 +1,79 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import fs from "fs";
+
+const mockCore = {
+  info: vi.fn(),
+  summary: {
+    addRaw: vi.fn().mockReturnThis(),
+    write: vi.fn().mockResolvedValue(),
+  },
+};
+
+global.core = mockCore;
+
+describe("generate_observability_summary.cjs", () => {
+  let module;
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    fs.mkdirSync("/tmp/gh-aw/mcp-logs", { recursive: true });
+    process.env.GH_AW_OBSERVABILITY_JOB_SUMMARY = "on";
+    module = await import("./generate_observability_summary.cjs");
+  });
+
+  afterEach(() => {
+    delete process.env.GH_AW_OBSERVABILITY_JOB_SUMMARY;
+    for (const path of ["/tmp/gh-aw/aw_info.json", "/tmp/gh-aw/agent_output.json", "/tmp/gh-aw/mcp-logs/gateway.jsonl", "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"]) {
+      if (fs.existsSync(path)) {
+        fs.unlinkSync(path);
+      }
+    }
+  });
+
+  it("builds summary from runtime observability files", async () => {
+    fs.writeFileSync(
+      "/tmp/gh-aw/aw_info.json",
+      JSON.stringify({
+        workflow_name: "triage-workflow",
+        engine_id: "copilot",
+        staged: false,
+        firewall_enabled: true,
+        context: { workflow_call_id: "trace-123" },
+      })
+    );
+    fs.writeFileSync(
+      "/tmp/gh-aw/agent_output.json",
+      JSON.stringify({
+        items: [{ type: "create_issue" }, { type: "add_comment" }],
+        errors: ["validation failed"],
+      })
+    );
+    fs.writeFileSync("/tmp/gh-aw/mcp-logs/gateway.jsonl", [JSON.stringify({ type: "DIFC_FILTERED" }), JSON.stringify({ type: "REQUEST" })].join("\n"));
+
+    await module.main(mockCore);
+
+    expect(mockCore.summary.addRaw).toHaveBeenCalledTimes(1);
+    const summary = mockCore.summary.addRaw.mock.calls[0][0];
+    expect(summary).toContain("<summary><b>Observability</b></summary>");
+    expect(summary).toContain("- **workflow**: triage-workflow");
+    expect(summary).toContain("- **engine**: copilot");
+    expect(summary).toContain("- **trace id**: trace-123");
+    expect(summary).toContain("- **posture**: write-capable");
+    expect(summary).toContain("- **created items**: 2");
+    expect(summary).toContain("- **blocked requests**: 1");
+    expect(summary).toContain("- **agent output errors**: 1");
+    expect(summary).toContain("  - add_comment");
+    expect(summary).toContain("  - create_issue");
+    expect(mockCore.summary.write).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips summary generation when opt-in mode is disabled", async () => {
+    process.env.GH_AW_OBSERVABILITY_JOB_SUMMARY = "off";
+
+    await module.main(mockCore);
+
+    expect(mockCore.summary.addRaw).not.toHaveBeenCalled();
+    expect(mockCore.summary.write).not.toHaveBeenCalled();
+    expect(mockCore.info).toHaveBeenCalledWith("Skipping observability summary: mode=off");
+  });
+});

cmd/gh-aw/main.go

@@ -718,6 +718,7 @@ Use "` + string(constants.CLIExtensionPrefix) + ` help all" to show help for all
 	logsCmd := cli.NewLogsCommand()
 	auditCmd := cli.NewAuditCommand()
 	healthCmd := cli.NewHealthCommand()
+	observabilityPolicyCmd := cli.NewObservabilityPolicyCommand()
 	mcpServerCmd := cli.NewMCPServerCommand()
 	prCmd := cli.NewPRCommand()
 	secretsCmd := cli.NewSecretsCommand()
@@ -760,6 +761,7 @@ Use "` + string(constants.CLIExtensionPrefix) + ` help all" to show help for all
 	logsCmd.GroupID = "analysis"
 	auditCmd.GroupID = "analysis"
 	healthCmd.GroupID = "analysis"
+	observabilityPolicyCmd.GroupID = "analysis"
 	checksCmd.GroupID = "analysis"
 
 	// Utilities
@@ -789,6 +791,7 @@ Use "` + string(constants.CLIExtensionPrefix) + ` help all" to show help for all
 	rootCmd.AddCommand(logsCmd)
 	rootCmd.AddCommand(auditCmd)
 	rootCmd.AddCommand(healthCmd)
+	rootCmd.AddCommand(observabilityPolicyCmd)
 	rootCmd.AddCommand(checksCmd)
 	rootCmd.AddCommand(mcpCmd)
 	rootCmd.AddCommand(mcpServerCmd)

pkg/cli/audit.go

@@ -340,6 +340,9 @@ func AuditWorkflowRun(ctx context.Context, runID int64, owner, repo, hostname st
 		fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to list artifacts: %v", err)))
 	}
 
+	currentCreatedItems := extractCreatedItemsFromManifest(runOutputDir)
+	run.SafeItemsCount = len(currentCreatedItems)
+
 	// Create processed run for report generation
 	processedRun := ProcessedRun{
 		Run:                     run,
@@ -352,8 +355,12 @@ func AuditWorkflowRun(ctx context.Context, runID int64, owner, repo, hostname st
 		JobDetails:              jobDetails,
 	}
 
+	currentSnapshot := buildAuditComparisonSnapshot(processedRun, currentCreatedItems)
+	comparison := buildAuditComparisonForRun(run, currentSnapshot, runOutputDir, owner, repo, hostname, verbose)
+
 	// Build structured audit data
 	auditData := buildAuditData(processedRun, metrics, mcpToolUsage)
+	auditData.Comparison = comparison
 
 	// Render output based on format preference
 	if jsonOutput {