From 7512585075a329d45f02b204d33133ab7292c9d9 Mon Sep 17 00:00:00 2001 From: Vichy Meas Date: Fri, 22 May 2026 21:19:49 +0000 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20async=20test=20stack=20cleanup=20?= =?UTF-8?q?=E2=80=94=20remove=20waiter=20+=20add=20sweeper=20Lambda?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove synchronous waiter from tearDown to prevent Lambda timeout kills. Add TestStackSweeperFunction to companion stack that runs every 6h to clean orphaned test stacks (with inline deep-clean for DELETE_FAILED) and orphaned CloudWatch log groups. --- integration/helpers/base_test.py | 2 - integration/setup/companion-stack.yaml | 207 +++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 2 deletions(-) diff --git a/integration/helpers/base_test.py b/integration/helpers/base_test.py index 720a8a25a..b2c9fcf2f 100644 --- a/integration/helpers/base_test.py +++ b/integration/helpers/base_test.py @@ -127,8 +127,6 @@ def tearDown(self): if self.stack_name: client = self.client_provider.cfn_client client.delete_stack(StackName=self.stack_name) - waiter = client.get_waiter("stack_delete_complete") - waiter.wait(StackName=self.stack_name) if self.output_file_path and os.path.exists(self.output_file_path): os.remove(self.output_file_path) if self.sub_input_file_path and os.path.exists(self.sub_input_file_path): diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml index 13a2be312..e3455be29 100644 --- a/integration/setup/companion-stack.yaml +++ b/integration/setup/companion-stack.yaml @@ -256,6 +256,213 @@ Resources: DependsOn: ApiGatewayLoggingRole Properties: CloudWatchRoleArn: !GetAtt ApiGatewayLoggingRole.Arn + + TestStackSweeperRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/ReadOnlyAccess" + Policies: + - PolicyName: TestStackSweeperPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - cloudformation:DeleteStack + - cloudformation:UpdateTerminationProtection + Resource: !Sub "arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/*sam-integ-stack-*" + - Effect: Allow + Action: + - iam:DeleteRolePolicy + - iam:DetachRolePolicy + - iam:DeleteRole + - iam:DeletePolicyVersion + - iam:DeletePolicy + Resource: + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/hydra-*" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/sam-integ-stack-*" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/hydra-*" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/sam-integ-stack-*" + - Effect: Allow + Action: + - s3:DeleteObject + - s3:DeleteObjectVersion + - s3:DeleteBucket + Resource: + - !Sub "arn:${AWS::Partition}:s3:::*sam-integ-stack-*" + - !Sub "arn:${AWS::Partition}:s3:::*sam-integ-stack-*/*" + - Effect: Allow + Action: + - logs:DeleteLogGroup + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*sam-integ-*:*" + - Effect: Allow + Action: + - lambda:DeleteEventSourceMapping + Resource: '*' + Condition: + StringEquals: + "lambda:createdBy": SAM + - Effect: Allow + Action: '*' + Resource: '*' + Condition: + StringLike: + "aws:ResourceTag/aws:cloudformation:stack-name": "*sam-integ-stack-*" + + TestStackSweeperFunction: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub "${AWS::StackName}-test-stack-sweeper" + Runtime: python3.10 + Handler: index.handler + Timeout: 900 + MemorySize: 256 + Role: !GetAtt TestStackSweeperRole.Arn + Code: + ZipFile: | + import boto3, time + from datetime import datetime, timezone, timedelta + + STACK_PATTERN = 'sam-integ-stack-' + IAM_PATTERN = 'sam-integ-' + ELIGIBLE_STATUSES = [ + 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'ROLLBACK_FAILED', + 'REVIEW_IN_PROGRESS', 'DELETE_FAILED', 'UPDATE_FAILED', + 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE', + 'UPDATE_ROLLBACK_FAILED'] + + def _has_time(ctx): + return ctx.get_remaining_time_in_millis() > 30000 + + def _is_test_resource(name): + return IAM_PATTERN in name and 'companion' not in name + + def _is_test_stack(name): + return STACK_PATTERN in name and 'companion' not in name + + def handler(event, ctx): + cfn = boto3.client('cloudformation') + iam = boto3.client('iam') + logs = boto3.client('logs') + cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + deleted = 0 + for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=ELIGIBLE_STATUSES): + for stack in page['StackSummaries']: + if not _has_time(ctx): + break + name = stack['StackName'] + if not _is_test_stack(name): + continue + if stack['CreationTime'].replace(tzinfo=timezone.utc) >= cutoff: + continue + try: + cfn.update_termination_protection(EnableTerminationProtection=False, StackName=name) + except: + pass + if stack['StackStatus'] == 'DELETE_FAILED': + _deep_clean(cfn, iam, name) + try: + cfn.delete_stack(StackName=name) + deleted += 1 + time.sleep(1) + except Exception as e: + print(f"delete_stack {name}: {e}") + print(f"Stacks: {deleted} deleted") + _clean_log_groups(logs, cutoff, ctx) + + def _deep_clean(cfn, iam, stack_name): + try: + events = cfn.describe_stack_events(StackName=stack_name)['StackEvents'] + for event in events: + if event.get('ResourceStatus') != 'DELETE_FAILED': + continue + resource_type = event.get('ResourceType', '') + resource_id = event.get('PhysicalResourceId', '') + if not resource_id or not _is_test_resource(resource_id): + continue + if resource_type == 'AWS::IAM::Role': + _delete_role(iam, resource_id) + elif resource_type == 'AWS::IAM::Policy': + _delete_policy(iam, resource_id) + elif resource_type == 'AWS::S3::Bucket': + _empty_bucket(resource_id) + except Exception as e: + print(f"deep_clean {stack_name}: {e}") + + def _delete_role(iam, role_name): + try: + for p in iam.list_role_policies(RoleName=role_name)['PolicyNames']: + iam.delete_role_policy(RoleName=role_name, PolicyName=p) + for p in iam.list_attached_role_policies(RoleName=role_name)['AttachedPolicies']: + iam.detach_role_policy(RoleName=role_name, PolicyArn=p['PolicyArn']) + iam.delete_role(RoleName=role_name) + except: pass + + def _delete_policy(iam, policy_arn): + try: + paginator = iam.get_paginator('list_entities_for_policy') + for pg in paginator.paginate(PolicyArn=policy_arn, EntityFilter='Role'): + for r in pg['PolicyRoles']: + iam.detach_role_policy(RoleName=r['RoleName'], PolicyArn=policy_arn) + for v in iam.list_policy_versions(PolicyArn=policy_arn)['Versions']: + if not v['IsDefaultVersion']: + iam.delete_policy_version(PolicyArn=policy_arn, VersionId=v['VersionId']) + iam.delete_policy(PolicyArn=policy_arn) + except: pass + + def _empty_bucket(bucket_name): + try: + bucket = boto3.resource('s3').Bucket(bucket_name) + bucket.object_versions.delete() + bucket.objects.delete() + except: pass + + def _clean_log_groups(logs, cutoff, ctx): + cutoff_ms = int(cutoff.timestamp() * 1000) + deleted = 0 + paginator = logs.get_paginator('describe_log_groups') + for page in paginator.paginate(logGroupNamePrefix='/aws/lambda/'): + for log_group in page['logGroups']: + if not _has_time(ctx): + return + name = log_group['logGroupName'] + if STACK_PATTERN not in name: + continue + if log_group.get('creationTime', 0) >= cutoff_ms: + continue + try: + logs.delete_log_group(logGroupName=name) + deleted += 1 + time.sleep(1) + except: pass + print(f"Log groups: {deleted} deleted") + + TestStackSweeperSchedule: + Type: AWS::Events::Rule + Properties: + ScheduleExpression: 'rate(6 hours)' + State: ENABLED + Targets: + - Arn: !GetAtt TestStackSweeperFunction.Arn + Id: TestStackSweeperTarget + + TestStackSweeperInvokePermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !GetAtt TestStackSweeperFunction.Arn + Action: lambda:InvokeFunction + Principal: events.amazonaws.com + SourceArn: !GetAtt TestStackSweeperSchedule.Arn + Outputs: PreCreatedVpc: Description: Pre-created VPC that can be used inside other tests From 4fd9bb58671c620733ab3224f93319938aa4424a Mon Sep 17 00:00:00 2001 From: Vichy Meas Date: Sat, 23 May 2026 05:35:01 +0000 Subject: [PATCH 2/5] update permission and refactor code --- integration/setup/companion-stack.yaml | 75 +++++++++++--------------- 1 file changed, 32 insertions(+), 43 deletions(-) diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml index e3455be29..994293a3f 100644 --- a/integration/setup/companion-stack.yaml +++ b/integration/setup/companion-stack.yaml @@ -304,19 +304,12 @@ Resources: Action: - logs:DeleteLogGroup Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*sam-integ-*:*" - - Effect: Allow - Action: - - lambda:DeleteEventSourceMapping - Resource: '*' - Condition: - StringEquals: - "lambda:createdBy": SAM - Effect: Allow Action: '*' Resource: '*' Condition: - StringLike: - "aws:ResourceTag/aws:cloudformation:stack-name": "*sam-integ-stack-*" + ForAnyValue:StringEquals: + "aws:CalledVia": cloudformation.amazonaws.com TestStackSweeperFunction: Type: AWS::Lambda::Function @@ -343,24 +336,27 @@ Resources: def _has_time(ctx): return ctx.get_remaining_time_in_millis() > 30000 - def _is_test_resource(name): - return IAM_PATTERN in name and 'companion' not in name - - def _is_test_stack(name): - return STACK_PATTERN in name and 'companion' not in name + def _is_test_resource(name, strict=False): + pattern = STACK_PATTERN if strict else IAM_PATTERN + return pattern in name and 'companion' not in name def handler(event, ctx): cfn = boto3.client('cloudformation') iam = boto3.client('iam') logs = boto3.client('logs') cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + + _sweep_stacks(cfn, iam, cutoff, ctx) + _sweep_log_groups(logs, cutoff, ctx) + + def _sweep_stacks(cfn, iam, cutoff, ctx): deleted = 0 for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=ELIGIBLE_STATUSES): for stack in page['StackSummaries']: if not _has_time(ctx): break name = stack['StackName'] - if not _is_test_stack(name): + if not _is_test_resource(name, strict=True): continue if stack['CreationTime'].replace(tzinfo=timezone.utc) >= cutoff: continue @@ -369,7 +365,7 @@ Resources: except: pass if stack['StackStatus'] == 'DELETE_FAILED': - _deep_clean(cfn, iam, name) + _fix_and_retry(cfn, iam, name) try: cfn.delete_stack(StackName=name) deleted += 1 @@ -377,12 +373,10 @@ Resources: except Exception as e: print(f"delete_stack {name}: {e}") print(f"Stacks: {deleted} deleted") - _clean_log_groups(logs, cutoff, ctx) - def _deep_clean(cfn, iam, stack_name): + def _fix_and_retry(cfn, iam, stack_name): try: - events = cfn.describe_stack_events(StackName=stack_name)['StackEvents'] - for event in events: + for event in cfn.describe_stack_events(StackName=stack_name)['StackEvents']: if event.get('ResourceStatus') != 'DELETE_FAILED': continue resource_type = event.get('ResourceType', '') @@ -390,15 +384,19 @@ Resources: if not resource_id or not _is_test_resource(resource_id): continue if resource_type == 'AWS::IAM::Role': - _delete_role(iam, resource_id) + _force_delete_role(iam, resource_id) elif resource_type == 'AWS::IAM::Policy': - _delete_policy(iam, resource_id) + _force_delete_policy(iam, resource_id) elif resource_type == 'AWS::S3::Bucket': - _empty_bucket(resource_id) + try: + bucket = boto3.resource('s3').Bucket(resource_id) + bucket.object_versions.delete() + bucket.objects.delete() + except: pass except Exception as e: - print(f"deep_clean {stack_name}: {e}") + print(f"fix_and_retry {stack_name}: {e}") - def _delete_role(iam, role_name): + def _force_delete_role(iam, role_name): try: for p in iam.list_role_policies(RoleName=role_name)['PolicyNames']: iam.delete_role_policy(RoleName=role_name, PolicyName=p) @@ -407,30 +405,21 @@ Resources: iam.delete_role(RoleName=role_name) except: pass - def _delete_policy(iam, policy_arn): + def _force_delete_policy(iam, arn): try: - paginator = iam.get_paginator('list_entities_for_policy') - for pg in paginator.paginate(PolicyArn=policy_arn, EntityFilter='Role'): - for r in pg['PolicyRoles']: - iam.detach_role_policy(RoleName=r['RoleName'], PolicyArn=policy_arn) - for v in iam.list_policy_versions(PolicyArn=policy_arn)['Versions']: + for page in iam.get_paginator('list_entities_for_policy').paginate(PolicyArn=arn, EntityFilter='Role'): + for r in page['PolicyRoles']: + iam.detach_role_policy(RoleName=r['RoleName'], PolicyArn=arn) + for v in iam.list_policy_versions(PolicyArn=arn)['Versions']: if not v['IsDefaultVersion']: - iam.delete_policy_version(PolicyArn=policy_arn, VersionId=v['VersionId']) - iam.delete_policy(PolicyArn=policy_arn) - except: pass - - def _empty_bucket(bucket_name): - try: - bucket = boto3.resource('s3').Bucket(bucket_name) - bucket.object_versions.delete() - bucket.objects.delete() + iam.delete_policy_version(PolicyArn=arn, VersionId=v['VersionId']) + iam.delete_policy(PolicyArn=arn) except: pass - def _clean_log_groups(logs, cutoff, ctx): + def _sweep_log_groups(logs, cutoff, ctx): cutoff_ms = int(cutoff.timestamp() * 1000) deleted = 0 - paginator = logs.get_paginator('describe_log_groups') - for page in paginator.paginate(logGroupNamePrefix='/aws/lambda/'): + for page in logs.get_paginator('describe_log_groups').paginate(logGroupNamePrefix='/aws/lambda/'): for log_group in page['logGroups']: if not _has_time(ctx): return From 83b2f1c9da134ce33da982c8528614276a14785f Mon Sep 17 00:00:00 2001 From: Vichy Meas Date: Tue, 26 May 2026 12:41:23 -0700 Subject: [PATCH 3/5] reformat --- integration/setup/companion-stack.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml index 994293a3f..c0ae30788 100644 --- a/integration/setup/companion-stack.yaml +++ b/integration/setup/companion-stack.yaml @@ -309,8 +309,8 @@ Resources: Resource: '*' Condition: ForAnyValue:StringEquals: - "aws:CalledVia": cloudformation.amazonaws.com - + aws:CalledVia: cloudformation.amazonaws.com + TestStackSweeperFunction: Type: AWS::Lambda::Function Properties: @@ -438,7 +438,7 @@ Resources: TestStackSweeperSchedule: Type: AWS::Events::Rule Properties: - ScheduleExpression: 'rate(6 hours)' + ScheduleExpression: rate(6 hours) State: ENABLED Targets: - Arn: !GetAtt TestStackSweeperFunction.Arn From 05e16ea67ffc097a6b183d3e0cd1d038a81e55b9 Mon Sep 17 00:00:00 2001 From: Vichy Meas Date: Tue, 26 May 2026 13:15:57 -0700 Subject: [PATCH 4/5] address comment: add print statement and ass exception --- integration/setup/companion-stack.yaml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml index c0ae30788..41d0a5862 100644 --- a/integration/setup/companion-stack.yaml +++ b/integration/setup/companion-stack.yaml @@ -327,7 +327,7 @@ Resources: STACK_PATTERN = 'sam-integ-stack-' IAM_PATTERN = 'sam-integ-' - ELIGIBLE_STATUSES = [ + STATUSES = [ 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'ROLLBACK_FAILED', 'REVIEW_IN_PROGRESS', 'DELETE_FAILED', 'UPDATE_FAILED', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE', @@ -336,7 +336,7 @@ Resources: def _has_time(ctx): return ctx.get_remaining_time_in_millis() > 30000 - def _is_test_resource(name, strict=False): + def _is_test(name, strict=False): pattern = STACK_PATTERN if strict else IAM_PATTERN return pattern in name and 'companion' not in name @@ -351,18 +351,19 @@ Resources: def _sweep_stacks(cfn, iam, cutoff, ctx): deleted = 0 - for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=ELIGIBLE_STATUSES): + for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=STATUSES): for stack in page['StackSummaries']: if not _has_time(ctx): - break + print(f"Deleted: {deleted}") + return name = stack['StackName'] - if not _is_test_resource(name, strict=True): + if not _is_test(name, strict=True): continue if stack['CreationTime'].replace(tzinfo=timezone.utc) >= cutoff: continue try: cfn.update_termination_protection(EnableTerminationProtection=False, StackName=name) - except: + except Exception: pass if stack['StackStatus'] == 'DELETE_FAILED': _fix_and_retry(cfn, iam, name) @@ -381,7 +382,7 @@ Resources: continue resource_type = event.get('ResourceType', '') resource_id = event.get('PhysicalResourceId', '') - if not resource_id or not _is_test_resource(resource_id): + if not resource_id or not _is_test(resource_id): continue if resource_type == 'AWS::IAM::Role': _force_delete_role(iam, resource_id) @@ -392,7 +393,7 @@ Resources: bucket = boto3.resource('s3').Bucket(resource_id) bucket.object_versions.delete() bucket.objects.delete() - except: pass + except Exception: pass except Exception as e: print(f"fix_and_retry {stack_name}: {e}") @@ -403,7 +404,7 @@ Resources: for p in iam.list_attached_role_policies(RoleName=role_name)['AttachedPolicies']: iam.detach_role_policy(RoleName=role_name, PolicyArn=p['PolicyArn']) iam.delete_role(RoleName=role_name) - except: pass + except Exception: pass def _force_delete_policy(iam, arn): try: @@ -414,7 +415,7 @@ Resources: if not v['IsDefaultVersion']: iam.delete_policy_version(PolicyArn=arn, VersionId=v['VersionId']) iam.delete_policy(PolicyArn=arn) - except: pass + except Exception: pass def _sweep_log_groups(logs, cutoff, ctx): cutoff_ms = int(cutoff.timestamp() * 1000) @@ -432,7 +433,7 @@ Resources: logs.delete_log_group(logGroupName=name) deleted += 1 time.sleep(1) - except: pass + except Exception: pass print(f"Log groups: {deleted} deleted") TestStackSweeperSchedule: From 9d8013556e27876c90f4cf064e5d1996528be8ec Mon Sep 17 00:00:00 2001 From: Vichy Meas Date: Tue, 26 May 2026 14:48:13 -0700 Subject: [PATCH 5/5] address review: python 3.13, remove termination protection, broaden log group cleanup --- integration/setup/companion-stack.yaml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml index 41d0a5862..5d4a39884 100644 --- a/integration/setup/companion-stack.yaml +++ b/integration/setup/companion-stack.yaml @@ -278,7 +278,6 @@ Resources: - Effect: Allow Action: - cloudformation:DeleteStack - - cloudformation:UpdateTerminationProtection Resource: !Sub "arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/*sam-integ-stack-*" - Effect: Allow Action: @@ -303,7 +302,7 @@ Resources: - Effect: Allow Action: - logs:DeleteLogGroup - Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*sam-integ-*:*" + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*sam-integ-*" - Effect: Allow Action: '*' Resource: '*' @@ -315,7 +314,7 @@ Resources: Type: AWS::Lambda::Function Properties: FunctionName: !Sub "${AWS::StackName}-test-stack-sweeper" - Runtime: python3.10 + Runtime: python3.13 Handler: index.handler Timeout: 900 MemorySize: 256 @@ -327,7 +326,7 @@ Resources: STACK_PATTERN = 'sam-integ-stack-' IAM_PATTERN = 'sam-integ-' - STATUSES = [ + ELIGIBLE_STATUSES = [ 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE', 'ROLLBACK_FAILED', 'REVIEW_IN_PROGRESS', 'DELETE_FAILED', 'UPDATE_FAILED', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE', @@ -350,30 +349,26 @@ Resources: _sweep_log_groups(logs, cutoff, ctx) def _sweep_stacks(cfn, iam, cutoff, ctx): - deleted = 0 - for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=STATUSES): + deleted = [] + for page in cfn.get_paginator('list_stacks').paginate(StackStatusFilter=ELIGIBLE_STATUSES): for stack in page['StackSummaries']: if not _has_time(ctx): - print(f"Deleted: {deleted}") + print(f"Attempt to delete ({len(deleted)}) stacks: {deleted}") return name = stack['StackName'] if not _is_test(name, strict=True): continue if stack['CreationTime'].replace(tzinfo=timezone.utc) >= cutoff: continue - try: - cfn.update_termination_protection(EnableTerminationProtection=False, StackName=name) - except Exception: - pass if stack['StackStatus'] == 'DELETE_FAILED': _fix_and_retry(cfn, iam, name) try: cfn.delete_stack(StackName=name) - deleted += 1 + deleted.append(name) time.sleep(1) except Exception as e: print(f"delete_stack {name}: {e}") - print(f"Stacks: {deleted} deleted") + print(f"Attempt to delete ({len(deleted)}) stacks: {deleted}") def _fix_and_retry(cfn, iam, stack_name): try: @@ -420,7 +415,7 @@ Resources: def _sweep_log_groups(logs, cutoff, ctx): cutoff_ms = int(cutoff.timestamp() * 1000) deleted = 0 - for page in logs.get_paginator('describe_log_groups').paginate(logGroupNamePrefix='/aws/lambda/'): + for page in logs.get_paginator('describe_log_groups').paginate(): for log_group in page['logGroups']: if not _has_time(ctx): return