🌐 AI搜索 & 代理 主页
Skip to content

Commit 078807d

Browse files
slapulaslapula
authored
Merge branch 'master' into s3-deploy
2 parents c7fccf9 + 7053c05 commit 078807d

File tree

9 files changed

+189
-78
lines changed

9 files changed

+189
-78
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,7 @@
2020
sha: v1.1.0
2121
hooks:
2222
- id: python-safety-dependencies-check
23+
- repo: https://github.com/asottile/add-trailing-comma
24+
sha: v0.6.4
25+
hooks:
26+
- id: add-trailing-comma

‎README.rst‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,8 @@ You can also choose to use S3 as your source for Lambda deployments. This can b
185185
Development
186186
===========
187187

188-
Development of this happens on GitHub, patches including tests, documentation are very welcome, as well as bug reports and feature contributions are welcome! Also please open an issue if this tool does not function as you'd expect.
188+
Development of "python-lambda" is facilitated exclusively on GitHub. Contributions in the form of patches, tests and feature creation and/or requests are very welcome and highly encouraged. Please open an issue if this tool does not function as you'd expect.
189+
189190

190191
How to release updates
191192
----------------------
@@ -196,10 +197,16 @@ Once complete, execute the following commands:
196197

197198
.. code:: bash
198199
199-
$ git checkout master
200-
$ bumpversion [major|minor|patch]
201-
$
202-
$ python setup.py sdist bdist_wheel upload
203-
$
204-
$ bumpversion --no-tag patch
205-
$ git push origin master --tags
200+
git checkout master
201+
202+
# Increment the version number and tag the release.
203+
bumpversion [major|minor|patch]
204+
205+
# Upload the distribution to PyPi
206+
python setup.py sdist bdist_wheel upload
207+
208+
# Since master often contains work-in-progress changes, increment the version
209+
# to a patch release to prevent inaccurate attribution.
210+
bumpversion --no-tag patch
211+
212+
git push origin master --tags

‎aws_lambda/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# flake8: noqa
33
__author__ = 'Nick Ficano'
44
__email__ = 'nficano@gmail.com'
5-
__version__ = '1.0.1'
5+
__version__ = '2.1.1'
66

77
from .aws_lambda import deploy, deploy_s3, invoke, init, build, upload, cleanup_old_versions
88

‎aws_lambda/aws_lambda.py‎

Lines changed: 114 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,35 @@
11
# -*- coding: utf-8 -*-
22
from __future__ import print_function
33

4+
import hashlib
45
import json
56
import logging
67
import os
78
import sys
89
import time
10+
from collections import defaultdict
911
from imp import load_source
1012
from shutil import copy
1113
from shutil import copyfile
14+
from shutil import copytree
1215
from tempfile import mkdtemp
1316

1417
import boto3
1518
import botocore
1619
import pip
1720
import yaml
18-
import hashlib
1921

2022
from .helpers import archive
23+
from .helpers import get_environment_variable_value
2124
from .helpers import mkdir
2225
from .helpers import read
2326
from .helpers import timestamp
24-
from .helpers import get_environment_variable_value
2527

2628

29+
ARN_PREFIXES = {
30+
'us-gov-west-1': 'aws-us-gov',
31+
}
32+
2733
log = logging.getLogger(__name__)
2834

2935

@@ -47,11 +53,13 @@ def cleanup_old_versions(src, keep_last_versions):
4753
aws_access_key_id = cfg.get('aws_access_key_id')
4854
aws_secret_access_key = cfg.get('aws_secret_access_key')
4955

50-
client = get_client('lambda', aws_access_key_id, aws_secret_access_key,
51-
cfg.get('region'))
56+
client = get_client(
57+
'lambda', aws_access_key_id, aws_secret_access_key,
58+
cfg.get('region'),
59+
)
5260

5361
response = client.list_versions_by_function(
54-
FunctionName=cfg.get('function_name')
62+
FunctionName=cfg.get('function_name'),
5563
)
5664
versions = response.get('Versions')
5765
if len(response.get('Versions')) < keep_last_versions:
@@ -63,7 +71,7 @@ def cleanup_old_versions(src, keep_last_versions):
6371
try:
6472
client.delete_function(
6573
FunctionName=cfg.get('function_name'),
66-
Qualifier=version_number
74+
Qualifier=version_number,
6775
)
6876
except botocore.exceptions.ClientError as e:
6977
print('Skipping Version {}: {}'
@@ -144,6 +152,7 @@ def upload(src, requirements=False, local_package=None):
144152

145153
upload_s3(cfg, path_to_zip_file)
146154

155+
147156
def invoke(src, alt_event=None, verbose=False):
148157
"""Simulates a call to your function.
149158
@@ -159,6 +168,11 @@ def invoke(src, alt_event=None, verbose=False):
159168
path_to_config_file = os.path.join(src, 'config.yaml')
160169
cfg = read(path_to_config_file, loader=yaml.load)
161170

171+
# Load environment variables from the config file into the actual
172+
# environment.
173+
for key, value in cfg.get('environment_variables').items():
174+
os.environ[key] = value
175+
162176
# Load and parse event file.
163177
if alt_event:
164178
path_to_event_file = os.path.join(src, alt_event)
@@ -200,7 +214,8 @@ def init(src, minimal=False):
200214
"""
201215

202216
templates_path = os.path.join(
203-
os.path.dirname(os.path.abspath(__file__)), 'project_templates')
217+
os.path.dirname(os.path.abspath(__file__)), 'project_templates',
218+
)
204219
for filename in os.listdir(templates_path):
205220
if (minimal and filename == 'event.json') or filename.endswith('.pyc'):
206221
continue
@@ -236,22 +251,41 @@ def build(src, requirements=False, local_package=None):
236251
output_filename = '{0}-{1}.zip'.format(timestamp(), function_name)
237252

238253
path_to_temp = mkdtemp(prefix='aws-lambda')
239-
pip_install_to_target(path_to_temp,
240-
requirements=requirements,
241-
local_package=local_package)
254+
pip_install_to_target(
255+
path_to_temp,
256+
requirements=requirements,
257+
local_package=local_package,
258+
)
242259

243260
# Hack for Zope.
244261
if 'zope' in os.listdir(path_to_temp):
245-
print('Zope packages detected; fixing Zope package paths to '
246-
'make them importable.')
262+
print(
263+
'Zope packages detected; fixing Zope package paths to '
264+
'make them importable.',
265+
)
247266
# Touch.
248267
with open(os.path.join(path_to_temp, 'zope/__init__.py'), 'wb'):
249268
pass
250269

251270
# Gracefully handle whether ".zip" was included in the filename or not.
252-
output_filename = ('{0}.zip'.format(output_filename)
253-
if not output_filename.endswith('.zip')
254-
else output_filename)
271+
output_filename = (
272+
'{0}.zip'.format(output_filename)
273+
if not output_filename.endswith('.zip')
274+
else output_filename
275+
)
276+
277+
# Allow definition of source code directories we want to build into our
278+
# zipped package.
279+
build_config = defaultdict(**cfg.get('build', {}))
280+
build_source_directories = build_config.get('source_directories', '')
281+
build_source_directories = (
282+
build_source_directories
283+
if build_source_directories is not None
284+
else ''
285+
)
286+
source_directories = [
287+
d.strip() for d in build_source_directories.split(',')
288+
]
255289

256290
files = []
257291
for filename in os.listdir(src):
@@ -262,14 +296,21 @@ def build(src, requirements=False, local_package=None):
262296
continue
263297
print('Bundling: %r' % filename)
264298
files.append(os.path.join(src, filename))
299+
elif os.path.isdir(filename) and filename in source_directories:
300+
print('Bundling directory: %r' % filename)
301+
files.append(os.path.join(src, filename))
265302

266303
# "cd" into `temp_path` directory.
267304
os.chdir(path_to_temp)
268305
for f in files:
269-
_, filename = os.path.split(f)
306+
if os.path.isfile(f):
307+
_, filename = os.path.split(f)
270308

271-
# Copy handler file into root of the packages folder.
272-
copyfile(f, os.path.join(path_to_temp, filename))
309+
# Copy handler file into root of the packages folder.
310+
copyfile(f, os.path.join(path_to_temp, filename))
311+
elif os.path.isdir(f):
312+
destination_folder = os.path.join(path_to_temp, f[len(src) + 1:])
313+
copytree(f, destination_folder)
273314

274315
# Zip them together into a single file.
275316
# TODO: Delete temp directory created once the archive has been compiled.
@@ -368,9 +409,10 @@ def pip_install_to_target(path, requirements=False, local_package=None):
368409
_install_packages(path, packages)
369410

370411

371-
def get_role_name(account_id, role):
412+
def get_role_name(region, account_id, role):
372413
"""Shortcut to insert the `account_id` and `role` into the iam string."""
373-
return 'arn:aws:iam::{0}:role/{1}'.format(account_id, role)
414+
prefix = ARN_PREFIXES.get(region, 'aws')
415+
return 'arn:{0}:iam::{1}:role/{2}'.format(prefix, account_id, role)
374416

375417

376418
def get_account_id(aws_access_key_id, aws_secret_access_key):
@@ -386,7 +428,7 @@ def get_client(client, aws_access_key_id, aws_secret_access_key, region=None):
386428
client,
387429
aws_access_key_id=aws_access_key_id,
388430
aws_secret_access_key=aws_secret_access_key,
389-
region_name=region
431+
region_name=region,
390432
)
391433

392434

@@ -399,10 +441,15 @@ def create_function(cfg, path_to_zip_file, *use_s3, **s3_file):
399441
aws_secret_access_key = cfg.get('aws_secret_access_key')
400442

401443
account_id = get_account_id(aws_access_key_id, aws_secret_access_key)
402-
role = get_role_name(account_id, cfg.get('role', 'lambda_basic_execution'))
444+
role = get_role_name(
445+
cfg.get('region'), account_id,
446+
cfg.get('role', 'lambda_basic_execution'),
447+
)
403448

404-
client = get_client('lambda', aws_access_key_id, aws_secret_access_key,
405-
cfg.get('region'))
449+
client = get_client(
450+
'lambda', aws_access_key_id, aws_secret_access_key,
451+
cfg.get('region'),
452+
)
406453

407454
# Do we prefer development variable over config?
408455
buck_name = (
@@ -448,8 +495,8 @@ def create_function(cfg, path_to_zip_file, *use_s3, **s3_file):
448495
key: get_environment_variable_value(value)
449496
for key, value
450497
in cfg.get('environment_variables').items()
451-
}
452-
}
498+
},
499+
},
453500
)
454501

455502
client.create_function(**kwargs)
@@ -464,10 +511,15 @@ def update_function(cfg, path_to_zip_file, *use_s3, **s3_file):
464511
aws_secret_access_key = cfg.get('aws_secret_access_key')
465512

466513
account_id = get_account_id(aws_access_key_id, aws_secret_access_key)
467-
role = get_role_name(account_id, cfg.get('role', 'lambda_basic_execution'))
514+
role = get_role_name(
515+
cfg.get('region'), account_id,
516+
cfg.get('role', 'lambda_basic_execution'),
517+
)
468518

469-
client = get_client('lambda', aws_access_key_id, aws_secret_access_key,
470-
cfg.get('region'))
519+
client = get_client(
520+
'lambda', aws_access_key_id, aws_secret_access_key,
521+
cfg.get('region'),
522+
)
471523

472524
# Do we prefer development variable over config?
473525
buck_name = (
@@ -497,8 +549,8 @@ def update_function(cfg, path_to_zip_file, *use_s3, **s3_file):
497549
'MemorySize': cfg.get('memory_size', 512),
498550
'VpcConfig': {
499551
'SubnetIds': cfg.get('subnet_ids', []),
500-
'SecurityGroupIds': cfg.get('security_group_ids', [])
501-
}
552+
'SecurityGroupIds': cfg.get('security_group_ids', []),
553+
},
502554
}
503555

504556
if 'environment_variables' in cfg:
@@ -508,8 +560,8 @@ def update_function(cfg, path_to_zip_file, *use_s3, **s3_file):
508560
key: get_environment_variable_value(value)
509561
for key, value
510562
in cfg.get('environment_variables').items()
511-
}
512-
}
563+
},
564+
},
513565
)
514566

515567
client.update_function_configuration(**kwargs)
@@ -520,17 +572,19 @@ def upload_s3(cfg, path_to_zip_file, *use_s3):
520572
print('Uploading your new Lambda function')
521573
aws_access_key_id = cfg.get('aws_access_key_id')
522574
aws_secret_access_key = cfg.get('aws_secret_access_key')
523-
account_id = get_account_id(aws_access_key_id, aws_secret_access_key)
524-
client = get_client('s3', aws_access_key_id, aws_secret_access_key,
525-
cfg.get('region'))
526-
role = get_role_name(account_id, cfg.get('role', 'basic_s3_upload'))
575+
client = get_client(
576+
's3', aws_access_key_id, aws_secret_access_key,
577+
cfg.get('region'),
578+
)
527579
byte_stream = b''
528580
with open(path_to_zip_file, mode='rb') as fh:
529581
byte_stream = fh.read()
530582
s3_key_prefix = cfg.get('s3_key_prefix', '/dist')
531583
checksum = hashlib.new('md5', byte_stream).hexdigest()
532584
timestamp = str(time.time())
533-
filename = '{prefix}{checksum}-{ts}.zip'.format(prefix=s3_key_prefix, checksum=checksum, ts=timestamp)
585+
filename = '{prefix}{checksum}-{ts}.zip'.format(
586+
prefix=s3_key_prefix, checksum=checksum, ts=timestamp,
587+
)
534588

535589
# Do we prefer development variable over config?
536590
buck_name = (
@@ -542,23 +596,37 @@ def upload_s3(cfg, path_to_zip_file, *use_s3):
542596
kwargs = {
543597
'Bucket': '{}'.format(buck_name),
544598
'Key': '{}'.format(filename),
545-
'Body': byte_stream
599+
'Body': byte_stream,
546600
}
547601

548602
client.put_object(**kwargs)
549603
print('Finished uploading {} to S3 bucket {}'.format(func_name, buck_name))
550604
if use_s3 == True:
551605
return filename
552606

607+
553608
def function_exists(cfg, function_name):
554609
"""Check whether a function exists or not"""
555610

556611
aws_access_key_id = cfg.get('aws_access_key_id')
557612
aws_secret_access_key = cfg.get('aws_secret_access_key')
558-
client = get_client('lambda', aws_access_key_id, aws_secret_access_key,
559-
cfg.get('region'))
560-
functions = client.list_functions().get('Functions', [])
561-
for fn in functions:
562-
if fn.get('FunctionName') == function_name:
563-
return True
564-
return False
613+
client = get_client(
614+
'lambda', aws_access_key_id, aws_secret_access_key,
615+
cfg.get('region'),
616+
)
617+
618+
# Need to loop through until we get all of the lambda functions returned.
619+
# It appears to be only returning 50 functions at a time.
620+
functions = []
621+
functions_resp = client.list_functions()
622+
functions.extend([
623+
f['FunctionName'] for f in functions_resp.get('Functions', [])
624+
])
625+
while('NextMarker' in functions_resp):
626+
functions_resp = client.list_functions(
627+
Marker=functions_resp.get('NextMarker'),
628+
)
629+
functions.extend([
630+
f['FunctionName'] for f in functions_resp.get('Functions', [])
631+
])
632+
return function_name in functions

‎aws_lambda/helpers.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# -*- coding: utf-8 -*-
2-
import os
3-
import zipfile
42
import datetime as dt
3+
import os
54
import re
5+
import zipfile
6+
67

78
def mkdir(path):
89
if not os.path.exists(path):

0 commit comments

Comments
 (0)