feat: agent secret scope

examples/secrets.py

@@ -0,0 +1,100 @@
+"""Example demonstrating the secret scope.
+
+Secrets let the agent *use* sensitive values (e.g. type a password) while the value is
+never sent to the LLM. The model only ever sees the placeholder
+``<|secret|>NAME<|secret|>``; the real value is substituted into tool calls at execution
+time. Literal values are also redacted from the LLM history, tool outputs and the cache.
+
+Two ways to provide a secret are shown:
+1. Hardcoded value (handy for a quick demo only).
+2. Read from an environment variable (recommended for real usage).
+
+Required environment variables (see .env):
+- ASKUI_WORKSPACE_ID, ASKUI_TOKEN - for the default AskUI providers
+- APP_PASSWORD - the example login password, read at runtime (see below)
+
+Set the secret in your shell before running (do NOT hardcode real secrets in code):
+    export APP_PASSWORD="my-real-password"
+
+Note: a secret typed into a *visible* field can still appear in screenshots sent to the
+model; on-screen secrets cannot currently be hidden.
+"""
+
+import logging
+import os
+
+from askui import ComputerAgent, Secret
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(levelname)s] %(asctime)s %(pathname)s:%(lineno)d | %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def secrets_from_env() -> list[Secret]:
+    """Build secrets by reading their values from environment variables.
+
+    This is the recommended approach: keep real values out of source code and pass them
+    in from the environment. We only *read* env vars here (never set them in code).
+    """
+    return [
+        Secret(
+            name="password",
+            value=os.environ["APP_PASSWORD"],
+            description="the application login password",
+        ),
+    ]
+
+
+def secrets_hardcoded() -> list[Secret]:
+    """Build secrets with hardcoded values.
+
+    Convenient for a quick local demo, but never commit real secrets to source control.
+    """
+    return [
+        Secret(
+            name="password",
+            value="hunter2-demo-only",
+            description="the application login password",
+        ),
+    ]
+
+
+def run_with_agent_level_secrets() -> None:
+    """Define secrets on the agent so they apply to every act()/type() call."""
+    with ComputerAgent(secrets=secrets_from_env()) as agent:
+        # The agent emits the placeholder; the real value is typed at execution time.
+        agent.act("Log in as 'admin' using the password")
+
+        # Deterministic typing also resolves the placeholder at the OS boundary.
+        agent.click("Password field")
+        agent.type("<|secret|>password<|secret|>")
+
+
+def run_with_per_call_secrets() -> None:
+    """Provide secrets only for a single act() call (overrides agent-level on name)."""
+    with ComputerAgent() as agent:
+        agent.act(
+            "Enter the one-time PIN into the verification field",
+            secrets=[
+                Secret(
+                    name="pin",
+                    value=os.environ.get("APP_OTP", "000000"),
+                    description="6-digit one-time PIN",
+                ),
+            ],
+        )
+
+
+def run_with_hardcoded_secret() -> None:
+    """Quick demo using a hardcoded secret value (not for production)."""
+    with ComputerAgent(secrets=secrets_hardcoded()) as agent:
+        agent.act("Log in using the password")
+
+
+if __name__ == "__main__":
+    # Pick the variant you want to try:
+    run_with_agent_level_secrets()
+    # run_with_per_call_secrets()
+    # run_with_hardcoded_secret()

src/askui/__init__.py

@@ -32,6 +32,7 @@
     UrlImageSourceParam,
 )
 from .models.exceptions import AutomationError
+from .models.shared.secrets import Secret, SecretVault
 from .models.shared.settings import (
     DEFAULT_GET_RESOLUTION,
     DEFAULT_LOCATE_RESOLUTION,
@@ -103,6 +104,8 @@
     "ResponseSchema",
     "ResponseSchemaBase",
     "Retry",
+    "Secret",
+    "SecretVault",
     "TextBlockParam",
     "TextCitationParam",
     "Tool",

src/askui/agent_base.py

@@ -15,6 +15,7 @@
 from askui.locators.locators import Locator
 from askui.models.shared.agent_message_param import MessageParam
 from askui.models.shared.conversation import Conversation, Speakers
+from askui.models.shared.secrets import Secret, SecretVault
 from askui.models.shared.settings import (
     ActSettings,
     CacheWritingSettings,
@@ -61,13 +62,20 @@ def __init__(
         settings: AgentSettings | None = None,
         callbacks: list[ConversationCallback] | None = None,
         truncation_strategy: TruncationStrategy | None = None,
+        secrets: list[Secret] | None = None,
     ) -> None:
         load_dotenv()
         self._reporter: Reporter = reporter or CompositeReporter(reporters=None)
         self._agent_os = agent_os
 
         self._tools = tools or []
 
+        # Secrets the agent may use but the LLM must never see. Real values are
+        # substituted into tool inputs at execution time; placeholders are all the
+        # model ever sees. Literal values are redacted from the LLM history and tool
+        # outputs. See `askui.models.shared.secrets`.
+        self._secret_vault = SecretVault(secrets)
+
         # Store settings and model providers
         _settings = settings or AgentSettings()
         self._vlm_provider = _settings.vlm_provider
@@ -117,7 +125,7 @@ def __init__(
         self.caching_settings = CachingSettings()
 
     @telemetry.record_call(
-        exclude={"goal", "act_settings", "tools", "tracing_settings"}
+        exclude={"goal", "act_settings", "tools", "tracing_settings", "secrets"}
     )
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def act(
@@ -127,6 +135,7 @@ def act(
         tools: list[Tool] | ToolCollection | None = None,
         caching_settings: CachingSettings | None = None,
         tracing_settings: OtelSettings | None = None,
+        secrets: list[Secret] | None = None,
     ) -> None:
         """
         Instructs the agent to achieve a specified goal through autonomous actions.
@@ -154,6 +163,14 @@ def act(
             tracing_settings (OtelSettings | None, optional): The tracing settings
                 for the act execution. Controls if and how traces are exported via
                 Opentelemetry.
+            secrets (list[Secret] | None, optional): Secrets available for this act
+                execution, in addition to any defined on the agent. The model only ever
+                sees the placeholder `<|secret|>NAME<|secret|>`; the real value is
+                substituted into tool inputs at execution time and is never sent to the
+                model. Per-call secrets override agent-level
+                secrets with the same name. Defaults to `None`. Note: a secret typed
+                into a visible field may still appear in screenshots sent to the model;
+                on-screen secrets cannot currently be hidden.
 
         Returns:
             None
@@ -228,18 +245,30 @@ def act(
                 # Agent can use existing caches and will record new actions
             ```
         """
+        # Merge agent-level and per-call secrets (per-call wins on name collision).
+        active_vault = self._secret_vault.merge(SecretVault(secrets))
+
         goal_str = (
             goal
             if isinstance(goal, str)
             else "\n".join(msg.model_dump_json() for msg in goal)
         )
-        self._reporter.add_message("User", f'act: "{goal_str}"')
+        # Redact any literal secret value the user may have placed in the goal before
+        # it reaches the reporter/logs.
+        redacted_goal_str = active_vault.redact(goal_str)
+        self._reporter.add_message("User", f'act: "{redacted_goal_str}"')
         logger.debug(
-            "Agent received instruction to act towards the goal '%s'", goal_str
+            "Agent received instruction to act towards the goal '%s'", redacted_goal_str
         )
         messages: list[MessageParam] = (
             [MessageParam(role="user", content=goal)] if isinstance(goal, str) else goal
         )
+        # Initial messages bypass Conversation._add_message, so redact them here to keep
+        # literal secrets out of the history sent to the LLM.
+        messages = [active_vault.redact_message(message) for message in messages]
+        # Make the vault available for substitution (tools) and redaction (history).
+        # The Conversation propagates it to the ToolCollection.
+        self._conversation.secret_vault = active_vault
         _act_settings = act_settings or self.act_settings
 
         _caching_settings: CachingSettings = caching_settings or self.caching_settings
@@ -275,6 +304,20 @@ def _build_tools(self, tools: list[Tool] | ToolCollection | None) -> ToolCollect
             tool_collection += tools
         return tool_collection
 
+    def _resolve_secrets(self, text: str) -> str:
+        """Substitute `<|secret|>NAME<|secret|>` placeholders with real values.
+
+        Used by deterministic input methods (e.g. `type`) so callers/agents can pass a
+        placeholder that resolves to the real value at the OS boundary.
+        """
+        resolved: str = self._secret_vault.substitute(text)
+        return resolved
+
+    def _redact_secrets(self, text: str) -> str:
+        """Redact literal secret values to their placeholders (for reporting/logs)."""
+        redacted: str = self._secret_vault.redact(text)
+        return redacted
+
     def _patch_act_with_cache(
         self,
         caching_settings: CachingSettings,

src/askui/android_agent.py

@@ -10,6 +10,7 @@
 from askui.container import telemetry
 from askui.locators.locators import Locator
 from askui.models.models import Point
+from askui.models.shared.secrets import Secret
 from askui.models.shared.settings import ActSettings, MessageSettings
 from askui.models.shared.tools import Tool
 from askui.models.shared.truncation_strategies import TruncationStrategy
@@ -53,6 +54,7 @@ class AndroidAgent(Agent):
         settings (AgentSettings | None, optional): Provider-based model settings. If `None`, uses the default AskUI model stack.
         retry (Retry, optional): The retry instance to use for retrying failed actions. Defaults to `ConfigurableRetry` with exponential backoff. Currently only supported for `locate()` method.
         act_tools (list[Tool] | None, optional): Additional tools to make available for the `act()` method.
+        secrets (list[Secret] | None, optional): Sensitive values (e.g. passwords) the agent may use but the LLM must never see. The model only sees the placeholder `<|secret|>NAME<|secret|>`; the real value is substituted at execution time and kept out of the LLM prompt, reporter, logs and cache. Also usable in deterministic `type()` and overridable per call via `act(..., secrets=[...])`. Note: a secret typed into a visible field may still appear in screenshots sent to the model; on-screen secrets cannot currently be hidden.
 
     Example:
         ```python
@@ -72,6 +74,7 @@ class AndroidAgent(Agent):
             "act_tools",
             "callbacks",
             "truncation_strategy",
+            "secrets",
         }
     )
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
@@ -84,6 +87,7 @@ def __init__(
         act_tools: list[Tool] | None = None,
         callbacks: list[ConversationCallback] | None = None,
         truncation_strategy: TruncationStrategy | None = None,
+        secrets: list[Secret] | None = None,
     ) -> None:
         reporter = CompositeReporter(reporters=reporters)
         self.os = PpadbAgentOs(device_identifier=device, reporter=reporter)
@@ -96,6 +100,7 @@ def __init__(
             settings=settings,
             callbacks=callbacks,
             truncation_strategy=truncation_strategy,
+            secrets=secrets,
         )
         self.act_tool_collection.add_agent_os(self.act_agent_os_facade)
         # Override default act settings with Android-specific settings
@@ -177,9 +182,10 @@ def type(
                 agent.type("password123")  # Types a password
             ```
         """
-        self._reporter.add_message("User", f'type: "{text}"')
-        logger.debug("AndroidAgent received instruction to type", extra={"text": text})
-        self.os.type(text)
+        # Reporter sees the placeholder; the device receives the resolved value.
+        self._reporter.add_message("User", f'type: "{self._redact_secrets(text)}"')
+        logger.debug("AndroidAgent received instruction to type")
+        self.os.type(self._resolve_secrets(text))
 
     @telemetry.record_call()
     @validate_call

src/askui/computer_agent.py

@@ -10,6 +10,7 @@
 from askui.container import telemetry
 from askui.locators.locators import Locator
 from askui.models.models import Point
+from askui.models.shared.secrets import Secret
 from askui.models.shared.settings import ActSettings, LocateSettings, MessageSettings
 from askui.models.shared.tools import Tool
 from askui.models.shared.truncation_strategies import TruncationStrategy
@@ -59,6 +60,13 @@ class ComputerAgent(Agent):
         act_tools (list[Tool] | None, optional): Additional tools to make available for
             the `act()` method for every call. Same tools can instead be passed per call
             via `act(..., tools=[...])` (see example below).
+        secrets (list[Secret] | None, optional): Sensitive values (e.g. passwords) the
+            agent may use but the LLM must never see. The model only sees the placeholder
+            `<|secret|>NAME<|secret|>`; the real value is substituted at execution time and is
+            kept out of the LLM prompt, reporter, logs and cache. Also usable in
+            deterministic `type()` and overridable per call via `act(..., secrets=[...])`.
+            Note: a secret typed into a visible field may still appear in screenshots sent
+            to the model; on-screen secrets cannot currently be hidden.
 
     Example:
         ```python
@@ -99,6 +107,7 @@ class ComputerAgent(Agent):
             "act_tools",
             "callbacks",
             "truncation_strategy",
+            "secrets",
         }
     )
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
@@ -112,6 +121,7 @@ def __init__(
         act_tools: list[Tool] | None = None,
         callbacks: list[ConversationCallback] | None = None,
         truncation_strategy: TruncationStrategy | None = None,
+        secrets: list[Secret] | None = None,
     ) -> None:
         reporter = CompositeReporter(reporters=reporters)
         self.tools = tools or AgentToolbox(
@@ -128,6 +138,7 @@ def __init__(
             settings=settings,
             callbacks=callbacks,
             truncation_strategy=truncation_strategy,
+            secrets=secrets,
         )
         self.act_agent_os_facade: ComputerAgentOsFacade = ComputerAgentOsFacade(
             self.tools.os
@@ -320,7 +331,9 @@ def type(
                 agent.type("text", locator="Input field", offset=(5, 0))  # Click 5 pixels right of "Input field", then type
             ```
         """
-        msg = f'type "{text}"'
+        # Reporter/logs see the placeholder; the OS receives the resolved value.
+        redacted_text = self._redact_secrets(text)
+        msg = f'type "{redacted_text}"'
         if locator is not None:
             msg += f" into {locator}"
             if clear:
@@ -337,7 +350,7 @@ def type(
             )
         logger.debug("Agent received instruction to %s", msg)
         self._reporter.add_message("User", msg)
-        self.tools.os.type(text)
+        self.tools.os.type(self._resolve_secrets(text))
 
     @telemetry.record_call()
     @validate_call