Fix: OutOfMemory error in mirror Lambda function (#7730)

.dockerignore

@@ -6,3 +6,13 @@
 !bin/keys/awscli-public-key.asc
 !bin/keys/docker-apt-keyring.pgp
 requirements.all.txt
+!bin/wheels/runtime
+!lambdas/layer/requirements.txt
+!lambdas/layer/requirements.trans.txt
+!src/azul
+!src/humancellatlas
+!lambdas/indexer/app.py
+!resources/static
+!lambdas/indexer/vendor/resources
+!lambdas/service/app.py
+!lambdas/service/vendor/resources

lambdas/Dockerfile

@@ -0,0 +1,20 @@
+FROM public.ecr.aws/lambda/python:3.14.2026.06.16.17
+
+ARG APP
+
+COPY bin/wheels/runtime/ /tmp/wheels/
+COPY lambdas/layer/requirements.txt lambdas/layer/requirements.trans.txt /tmp/
+RUN sed -e '/^--find-links/d' -e '/^chalice==/d' /tmp/requirements.txt > /tmp/requirements.clean.txt \
+    && pip install --no-index --find-links /tmp/wheels -r /tmp/requirements.clean.txt \
+    && pip install --no-deps --no-index --find-links /tmp/wheels chalice==1.32.0+21 \
+    && rm -rf /tmp/wheels /tmp/requirements*.txt
+
+COPY src/azul ${LAMBDA_TASK_ROOT}/azul
+COPY src/humancellatlas ${LAMBDA_TASK_ROOT}/humancellatlas
+COPY lambdas/${APP}/vendor/resources ${LAMBDA_TASK_ROOT}/resources
+RUN rm -rf ${LAMBDA_TASK_ROOT}/resources/static
+COPY resources/static ${LAMBDA_TASK_ROOT}/resources/static
+RUN chmod -R a+r ${LAMBDA_TASK_ROOT}/resources
+COPY lambdas/${APP}/app.py ${LAMBDA_TASK_ROOT}/app.py
+
+CMD ["app.app"]

lambdas/Makefile

@@ -1,12 +1,16 @@
 .PHONY: all
-all: layer indexer service
+all: layer registry indexer service
 
 include ../common.mk
 
 .PHONY: layer
 layer: check_env
 	$(MAKE) -C layer layer
 
+.PHONY: registry
+registry:
+	$(MAKE) -C $(project_root)/terraform create_registry
+
 .PHONY: indexer
 indexer: check_env
 	$(MAKE) -C indexer package

lambdas/indexer/Makefile

@@ -2,3 +2,21 @@
 all: package
 
 include ../lambdas.mk
+
+package: image_push
+
+image_ref := $(azul_docker_registry)$(AZUL_RESOURCE_PREFIX)-indexer-$(AZUL_DEPLOYMENT_STAGE):$(shell git -C $(project_root) rev-parse HEAD | cut -c1-12)
+
+.PHONY: image
+image:
+	aws ecr-public get-login-password | docker login --password-stdin --username AWS public.ecr.aws
+	docker build \
+	    --platform linux/amd64 \
+	    --build-arg APP=indexer \
+	    -f $(project_root)/lambdas/Dockerfile \
+	    -t $(image_ref) \
+	    $(project_root)
+
+.PHONY: image_push
+image_push: image
+	docker push $(image_ref)

lambdas/service/Makefile

@@ -6,4 +6,20 @@ include ../lambdas.mk
 .PHONY: apat_public_key
 apat_public_key: vendor/resources/apat_public_key.json
 
-package: apat_public_key
+package: apat_public_key image_push
+
+image_ref := $(azul_docker_registry)$(AZUL_RESOURCE_PREFIX)-service-$(AZUL_DEPLOYMENT_STAGE):$(shell git -C $(project_root) rev-parse HEAD | cut -c1-12)
+
+.PHONY: image
+image:
+	aws ecr-public get-login-password | docker login --password-stdin --username AWS public.ecr.aws
+	docker build \
+	    --platform linux/amd64 \
+	    --build-arg APP=service \
+	    -f $(project_root)/lambdas/Dockerfile \
+	    -t $(image_ref) \
+	    $(project_root)
+
+.PHONY: image_push
+image_push: image
+	docker push $(image_ref)

requirements.all.txt

@@ -6,7 +6,7 @@ boto3==1.43.27
 boto3-stubs-lite==1.43.27
 botocore==1.43.27
 botocore-stubs==1.43.14
-certifi==2026.5.20
+certifi==2026.6.17
 cffi==2.0.0
 chalice==1.32.0+21
 charset-normalizer==3.4.7
@@ -35,7 +35,7 @@ google-cloud-storage==3.11.0
 google-crc32c==1.8.0
 google-resumable-media==2.10.0
 googleapis-common-protos==1.75.0
-greenlet==3.5.1
+greenlet==3.5.2
 grpc-google-iam-v1==0.14.4
 grpcio==1.81.1
 grpcio-status==1.81.1
@@ -63,12 +63,12 @@ mypy==2.1.0
 mypy-boto3-apigateway==1.43.0
 mypy-boto3-cloudwatch==1.43.26
 mypy-boto3-dynamodb==1.43.0
-mypy-boto3-ec2==1.43.27
+mypy-boto3-ec2==1.43.35
 mypy-boto3-ecr==1.43.0
 mypy-boto3-iam==1.43.29
 mypy-boto3-kms==1.43.12
-mypy-boto3-lambda==1.43.20
-mypy-boto3-opensearch==1.43.16
+mypy-boto3-lambda==1.43.35
+mypy-boto3-opensearch==1.43.34
 mypy-boto3-s3==1.43.31
 mypy-boto3-secretsmanager==1.43.0
 mypy-boto3-securityhub==1.43.5
@@ -120,7 +120,7 @@ setuptools==82.0.1
 six==1.17.0
 smmap==5.0.3
 strict-rfc3339==0.7
-tqdm==4.68.2
+tqdm==4.68.3
 types-awscrt==0.34.1
 types-chevron==0.14.2
 types-jsonschema==4.26.0.20260518

requirements.dev.trans.txt

@@ -1,29 +1,24 @@
 ast_serialize==0.5.0
-blessed==1.44.0
 botocore-stubs==1.43.14
-click==8.4.1
-editor==1.8.0
 et_xmlfile==2.0.0
 gitdb==4.0.12
 google-auth-httplib2==0.4.0
-greenlet==3.5.1
+greenlet==3.5.2
 grpc-google-iam-v1==0.14.4
 httplib2==0.31.2
-inquirer==3.4.1
-jinxed==2.0.4
 jsonschema-path==0.3.4
 lazy-object-proxy==1.12.0
 librt==0.11.0
 mccabe==0.7.0
 mypy-boto3-apigateway==1.43.0
 mypy-boto3-cloudwatch==1.43.26
 mypy-boto3-dynamodb==1.43.0
-mypy-boto3-ec2==1.43.27
+mypy-boto3-ec2==1.43.35
 mypy-boto3-ecr==1.43.0
 mypy-boto3-iam==1.43.29
 mypy-boto3-kms==1.43.12
-mypy-boto3-lambda==1.43.20
-mypy-boto3-opensearch==1.43.16
+mypy-boto3-lambda==1.43.35
+mypy-boto3-opensearch==1.43.34
 mypy-boto3-s3==1.43.31
 mypy-boto3-secretsmanager==1.43.0
 mypy-boto3-securityhub==1.43.5
@@ -41,18 +36,14 @@ pycodestyle==2.14.0
 pyflakes==3.4.0
 pynacl==1.6.2
 pyparsing==3.3.2
-readchar==4.2.2
 requests-toolbelt==1.0.0
 rfc3339-validator==0.1.4
-runs==1.3.0
 smmap==5.0.3
-tqdm==4.68.2
+tqdm==4.68.3
 types-awscrt==0.34.1
 types-s3transfer==0.16.0
 uritemplate==4.2.0
-wcwidth==0.8.1
 www-authenticate==0.9.2
 xmltodict==1.0.4
-xmod==1.10.0
 zope.event==6.2
 zope.interface==8.5

requirements.dev.txt

@@ -2,7 +2,6 @@
 --find-links https://github.com/DataBiosphere/azul-requirements-parser/releases/expanded_assets/v0.13.0+6  # match version of requirements-parser requirement below
 atomicwrites==1.4.1
 boto3-stubs-lite[apigateway,cloudwatch,dynamodb,ec2,ecr,iam,kms,lambda,opensearch,s3,secretsmanager,securityhub,sns,sqs,ssm,stepfunctions,sts]==1.43.27  # match this with the version of the `boto3` runtime dendency
-chalice==1.32.0+21  # match version with find-links entry above
 coverage==7.14.1
 docker==7.1.0
 flake8==7.3.0

requirements.trans.txt

@@ -1,6 +1,9 @@
-certifi==2026.5.20
+blessed==1.44.0
+certifi==2026.6.17
 cffi==2.0.0
 charset-normalizer==3.4.7
+click==8.4.1
+editor==1.8.0
 events==0.5
 google-cloud-core==2.6.0
 google-crc32c==1.8.0
@@ -9,6 +12,8 @@ googleapis-common-protos==1.75.0
 grpcio==1.81.1
 grpcio-status==1.81.1
 idna==3.18
+inquirer==3.4.1
+jinxed==2.0.4
 jsonschema-specifications==2025.9.1
 markupsafe==3.0.3
 orderedmultidict==1.0.2
@@ -20,7 +25,12 @@ pyasn1_modules==0.4.2
 pycparser==3.0
 pyopenssl==26.2.0
 python-dateutil==2.9.0.post0
+pyyaml==6.0.3
+readchar==4.2.2
 rpds-py==2026.5.1
+runs==1.3.0
 s3transfer==0.18.0
 six==1.17.0
 typing_extensions==4.15.0
+wcwidth==0.8.1
+xmod==1.10.0

requirements.txt

@@ -1,5 +1,7 @@
+--find-links https://github.com/DataBiosphere/azul-chalice/releases/expanded_assets/v1.32.0+21  # match version of chalice requirement below
 --find-links  https://github.com/DataBiosphere/azul-resumablehash/releases/expanded_assets/1.5  # match version of resumablehash requirement below
 attrs==26.1.0
+chalice==1.32.0+21  # match version with find-links entry above
 boto3==1.43.27  # Match version of the `boto3-stubs` dev dependency
 botocore==1.43.27
 chevron==0.14.0  # Match with types-chevron in requirements.dev.txt

src/azul/infra/terraform.py

@@ -762,9 +762,10 @@ def resource_items(resource_type: str) -> Iterable[tuple[str, MutableJSON]]:
                 '${aws_vpc_endpoint.%s.id}' % app_name
             ]
 
+        commit = config.git_status['commit'][:12]
+        image_uri = f'{config.docker_registry}/{config.qualified_resource_name(app_name)}:{commit}'
         for resource_name, resource in resource_items('aws_lambda_function'):
             assert 'layers' not in resource
-            resource['layers'] = ['${aws_lambda_layer_version.dependencies.arn}']
             # Publishing a new Lambda function version each time lets us perform
             # an atomic update of the function, avoiding a race condition
             # between the update of the function's configuration and its code.
@@ -782,9 +783,13 @@ def resource_items(resource_type: str) -> Iterable[tuple[str, MutableJSON]]:
                 )
             )
             json_dict(json_dict(resource['environment'])['variables']).update(env)
-            package_zip = str(self.package_zip_path(app_name))
-            resource['source_code_hash'] = '${filebase64sha256("%s")}' % package_zip
-            resource['filename'] = package_zip
+            handler = resource.pop('handler')
+            del resource['runtime']
+            del resource['source_code_hash']
+            del resource['filename']
+            resource['package_type'] = 'Image'
+            resource['image_uri'] = image_uri
+            resource['image_config'] = {'command': [handler]}
 
         # Replace any references to unqualified function ARNs in the OpenAPI
         # spec emitted by Chalice with references to the alias.
@@ -1013,21 +1018,6 @@ def alias_property(property_name: str, resource: MutableJSON):
             assert 'lifecycle' not in resource, (resource_name, resource)
             resource['lifecycle'] = {'create_before_destroy': True}
 
-        if config.lambda_runtime_version is not None:
-            resource_type = 'aws_lambda_runtime_management_config'
-            runtime_version = config.lambda_runtime_version
-            assert isinstance(runtime_version, str), runtime_version
-            runtime_version_configs: MutableJSON = {}
-            for resource_name, resource in resource_items('aws_lambda_function'):
-                runtime_version_configs[resource_name] = {
-                    'function_name': '${aws_lambda_function.%s.function_name}' % resource_name,
-                    'qualifier': '${aws_lambda_function.%s.version}' % resource_name,
-                    'update_runtime_on': 'Manual',
-                    'runtime_version_arn': 'arn:aws:lambda:us-east-1::runtime:' + runtime_version,
-                }
-            assert resource_type not in resources, resources
-            resources[resource_type] = runtime_version_configs
-
         resource_type = 'aws_lambda_function_recursion_config'
         recursion_configs: MutableJSON = {}
         for resource_name, resource in resource_items('aws_lambda_function'):

src/azul/logging.py

@@ -2,6 +2,8 @@
     contextmanager,
 )
 import logging
+import os
+import sys
 from typing import (
     Any,
     Literal,
@@ -84,21 +86,26 @@ def configure_app_logging(app: AzulChaliceApp, *loggers):
         # Environment is not unit test
         root_logger = logging.getLogger()
         if root_logger.hasHandlers():
-            # If a handler is already present, we're running on AWS Lambda. See
+            # A handler is already present. On Lambda, awslambdaric sets this up
+            # before module import in some versions. See
             #
             # https://github.com/aws/aws-lambda-python-runtime-interface-client/blob/3f43f4d0/awslambdaric/bootstrap.py#L454
             #
-            # for details.
-            #
             handler = one(root_logger.handlers)
-            root_formatter = logging.Formatter(lambda_log_format, lambda_log_date_format)
-            handler.setFormatter(root_formatter)
         else:
-            # Otherwise, we're running `chalice local`
-            handler = logging.StreamHandler()
+            # No handler yet: either `chalice local`, or Lambda with a newer
+            # awslambdaric that defers handler setup until after module import.
+            # Use stdout on Lambda because it is always captured to CloudWatch.
+            on_lambda = bool(os.environ.get('AWS_LAMBDA_RUNTIME_API'))
+            handler = logging.StreamHandler(sys.stdout if on_lambda else sys.stderr)
             logging.basicConfig(format=lambda_log_format,
                                 datefmt=lambda_log_date_format,
                                 handlers=[handler])
+        # Explicitly clear any level restriction set by awslambdaric so that our
+        # log-level configuration fully controls which records are emitted.
+        handler.setLevel(logging.NOTSET)
+        root_formatter = logging.Formatter(lambda_log_format, lambda_log_date_format)
+        handler.setFormatter(root_formatter)
         handler.addFilter(LambdaLogFilter(app))
 
 

terraform/Makefile

@@ -36,6 +36,10 @@ validate: config
 rename_resources: validate
 	python $(project_root)/scripts/rename_resources.py
 
+.PHONY: create_registry
+create_registry: validate
+	terraform apply -target aws_ecr_repository.indexer_lambda -target aws_ecr_repository.service_lambda
+
 .PHONY: import_resources
 import_resources: rename_resources
 

terraform/api_gateway.tf.json.template.py

@@ -795,7 +795,12 @@ def add_waf_blocked_alarm(resources: JSON) -> JSON:
                 resource_name: {
                     'name': config.active_function_alias_name,
                     'function_name': '${aws_lambda_function.%s.function_name}' % resource_name,
-                    'function_version': '${aws_lambda_function.%s.version}' % resource_name
+                    'function_version': '${aws_lambda_function.%s.version}' % resource_name,
+                    'lifecycle': {
+                        'replace_triggered_by': [
+                            'aws_lambda_function.%s.package_type' % resource_name
+                        ]
+                    }
                 }
                 for app in apps
                 for resource_name in app.chalice.tf_function_resource_names

terraform/lambda_ecr.tf.json.template.py

@@ -0,0 +1,18 @@
+from azul import (
+    config,
+)
+from azul.infra.terraform import (
+    emit_tf,
+)
+
+emit_tf({
+    'resource': {
+        'aws_ecr_repository': {
+            f'{app}_lambda': {
+                'name': config.qualified_resource_name(app),
+                'force_delete': True
+            }
+            for app in ('indexer', 'service')
+        }
+    }
+})