#!/usr/bin/env python
"""
/* This file is under the Modified MIT License.  
 *
 * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
------------------------------------------------------------------------------

This Lambda function is set up as a subscription to a Cloudwatch Logs group. 
As new logs are written to the log group, they're sent to this Lambda, which unzips them,
pulls out the log messages, and writes them to the desired S3 bucket and key prefix.

The filename convention is: the log stream name + a timestamp.

Tested with: 
* EC2 logs
* Security changes

Set up the following as Lambda environment variables:
    s3BucketName         s3 bucket name, to write files to
    s3KeyPrefix          prefix to put in front of file names (for this log group)
    
They can be set via console, CLI or CF template. E.g.:
        aws lambda create-function \
          --region us-east-1
          --function-name myFunction
          --zip-file file://path/package.zip
          --role role-arn
          --environment Variables={bucket1=mybucket1}
          --handler index.handler
          --runtime python2.7
          --profile default   

@author V.Megler
@date August 2017
Updated 2017-12 for Python 3.9          
"""
import boto3
import logging
import json
import gzip
import time
import datetime
import os
from io import BytesIO
import base64
from botocore.exceptions import ClientError

# Sample event
event = { "awslogs": { "data": "H4sIAAAAAAAAAN1da2/bOBb9K0S+uMXGtiTLlm1MB8g2bqe7fQBNiu6iCAJaom1tJFEjUU49Rf/7XurhV+Q2sUiXnAE6rV6XOpfn8t5Dida3s5CkKZ6T61VMzsZnlxfXF7fvJldXF68nZ+dn9D4iCewemtagPxr1nb5lwe6Azl8nNIvhyOSl9ZbO02LnFUsIDmFvkkW9Nhx6E6UMRy65Yti9a5v/spw/Xn+4cP7zh3HZ9dvGzIP/pj17apu4NxsOu0ucdMFO1w1o5rX9yGcd2ATjaTZN3cSPmU+jV37ASJKejb+cRXSWb5zd5O1PliRi/MC3M9+D2+j1erZpwK07Q8eAf5o9a9S3LXto9owhHDRGzmhgwsH+cDByHNOGvXbPsKFB5oNjGA4Bo2mP+sYATh8ZhnFeOQzMv8MrZDrI6o0ta2yO0Oauv1i2Y9+MUcb8oBOvvlxO/vnpNWy/XJ+Blh1kdJzOAIGvIj+aoxbf3Q6oi4MWwgx9Jt45N89bsYztdv4B92F00KcY9TtDC6XEpZGXds6+nzfD3ZeG+3PiM46RUXSgj1Eb4ekYfRnY9g0y0HTFSNoY0EAaoAvGSBhXmBIS0iUpofnTAlrXL8nfnVLK2jMwly6I1xiUowSoxjCGvxSGhxnuRrQdEdYYyUgako8EexzGLKHhQXbR6f868V2Anv2Z+YS9eIWDlDxvCso0hIECC3OSbsP6lHJQnp+yhMIFOE3Rb8VfrfzyfEwoDqcdHOK/aNS5zDdbvzcGZgoDdnvLt25vt6G9pfQu7zGa5H84z1BKs8QlyI9gfGtNXKt1jlrvaURaN+do6WMUQ3LkPoKj/NDGB8WFaesGsQVkhBAzdwGneSQmkUci18+vefXm7eTqv1fXk3etm8busaS654rgxF1UDnrgnOa33xN2+1t7qrt/T1GenvMbL+97RrOo8Zhuyqs4doeQOKFuN4t5U4LHC3m1A0eAzLIkeIij8Z3LKxI2Z6CQeqSo8FqQlOgd1H6G7VSlG3oGm/3mnSCkNBhaxxSujy5ZBx3DEFaymkKqiFrEv6ZkNYUUE7WAPpbd5dIwxJEHicOfQS/M/DlPOm0MWebeZwuEg4DeEw8KJ5YlEZzu8Sxj3KBnUL4GQTFgnCMXx3CcvLhOsubDhyWk3HgkbKj5oBmWo06iXwpbSDGSw65JVy8XxL0rKZwSSLEzBGoZUOXFxD1BEQG8OADN7q0Q+QoVVjG8QkWSkKVPs5THeHk6zQKvcBKC/dwkozGCfB6sGoexJaTqqO/+H5TQWyJAbDa0hJQhT8YjVxJYQoqUQ2SdfCVuxmd3IPIiGHgzAHmOIrpLRY8w4jJg7Qk4KaSiUa0PhRQ7OSi1ZJ0lrvRRUNZxw5Przx8+/ru5wrPElUxHKLzWJey5yneUHtvsKHzXGJ+4CqoeH+HgIJdCAnVxhOaQPHC0KqDmEf2Q9GWHd3bA7241D4GexBrqBwMWVBZel7hWNyQMt3MvCB2yeuJqpF337+Djc4Q8q3BBz/PJGi1HlYPKkiDlBF4wFo+763E6R9wZDx2n15y7PYF1UBLcQt0ak2Qb6Beja94gCuMMqoCYg1HH6tud8u9ugEHGsE1vbpD6Xqsolb+1wHprfJwJCPkF0IkkKVj41vqUkqR9MScR4wYLmfkGcHRzjdn6DqfnCf42IZ6fQPLnl/HiGw5AAwvq8eteT665Ye46mnFLfRDU31Ehb7IE88KicecILOpqOyef6MhJd4Rf0TML7gKZo+lzhGeMJMgERZ5PhDcWpT1xld/BECyKh3W0FSPmQYq1GkMSWN0dirTB4yIt4xHAQT8tujaXnSiixAeUuGK0WUCtXVkGkdU3DBlhJHB6riHnNkPIsUO6vqwTOGUoaBivaGc5MlgncEZRGOumAXXv2h4oe0heIY5jGPyPpmK9MV35aQss4wXxs9bBJWmHEihriyv5JVMWh75Q1nJ72hJXtngRRFzwccldGbWyLVslCONuQikTSt7coLbsFadxpLKXO1kmfdXRRVsyF4dHFwf5tdqSUhWFtOvPql4dyCCgiiKJg/ajGW1Cwvx6bYmonmiqfLrW7LYMNqoonuBfie+mRw+J6+t1ZWNfPYlU+bRkowwh31dRFVW4lwuasvDoBwT7ZrRlpnoaaM+1JUF7MnR7X0XtExF2T5O7o0fL9fXaclI9ZVP5tHpoZcogo4pKpgLuRwB2hl1yfBavM6UtRdXTOTXuLdnal0FWFVVPjQtC3KDuPGhPW9qqp4oO+bgaaaWkfRVF0iFHGIMx8cbubDztjw0yNmzhdK5pQVeCD9QTWo/3+prylgTOD1QUY4/3Tb6Mrc0VQYRDIj8C9trTNh7Uk3fH9kEZHbaMF10GKuvAnzsqJW6W+GzVnvPfuEjlR8d+g9qGh7pK88mdUMWHlOShiTQ94Kll7Lb9eGm3Xd9L2vlTyhMESW2r2kaKFoL3CT0hcyZnoJM4rhlYsimc3ODt+WOa0jYw9JHUP3B/FQ0ypooGeqvtmsHjl+QObSPE0VmT13SEzMzh6K3S12ecJHnstKZtdOis0Ld7oAoLGfrD0Vufw2750cAb0TYIdNbhsLvivoy3aBy9tffuLAWMEyefnsrb1DYydNbdD/tBau2kt+ouZrt5rXmCCNluTNvQ0Fl5b3VAFRMyHmw4emvv/Ne8T6Il1i3pGg1DnVV25X2ZGmKot7SupupOPRN1oF1tw0RnuV3fF1KDRm/hXS5FjLJwShL5sbLbnLYhorMY3+mCKjJkBIbeqjyf1j5BWVW2o20o6Ky+C99LzQ4qKu44AKeEwK2j35PdsqAtcdXTxhuvypwtHaooeDfQ8RL7AZ76AZ8V+4v/fGhjjj40qStpR+pJ2B+4Wea0zUhFrQo04rXV8ctpNga0Zah66nHt1IqPMn7jYqSiDFwj92iI/agxK0sz2nJTPdm259qKoT0ZDFVRj63xxzhhPu/kxiTdWNKWp+ppqoferX50QAZTlRRNCf8u6fHVaHm5tpxUUC4VLpX5wHykolbaPBg9loxbFjTlo22op4Q2XpUofGxDReGTEJ4g8o5tMGm6Z0VbaqongXY9K/G7Arahog4SszT677Hg2RbzKXQJI+cJljHbYr6HLlwECVmc/DdZcmyL+cS7YPlzqoXEtphvwQtmKA79doCzyF1AAx75eixFH9jRlqPqyaF938p7tG8bKsqipuld+8RuqieJTpHSTRUFEQ/G44VQebW2RFRPABUelfimh22qKHzWH5PDbpO59X0z2hJTPemz59pqoJRBUBWlT4N1nxqv5rRN9SSO3DWatqmqrAlx5M9gdzvGbNEkYe8a0paZagqbHefK+1aObaqobNY5gq3io+XNrhFd6Wmpp3F2HCvvC3q2paLO0fND0uJpqZ7iOc13pG1LHdnjrSJIFe42cmCOz1ZPY+ZhM9qyUxXZc9C11ctEEt4msi11dM9B/F2Puhl/iVoQVTf2tOWsKgrp5z5ePw2SMr6qo5sOeyL15xFmWfLE4vQxBrXlryo66hFOXut+GdNOljqK6rAr4js3dUSRtzCmK3F7qiisnzh4/Y5dX0bR0FNHbB32Q5Jiy7CHonhbmdOWuaqIsJ+6eL3MYyCjZOiJk2SXoCSvaJa4ZOJa2654meD7ANEZ4oKT601UrhBAjNI7ZMAm9K3XHIs46QIW5iTlIN68f/UBdrylQHIP8btPc4y7cFF7d7sxFIE6hPnBPjH9aF5wk78X73bd0At8vgTzz8wn7MUrHKTkeWMIAovyGgiob6DpCkKoBkjjWxdYR//Q+4TBPfNriv933NlccCcIrCxrO6E/2umFGkCNIQisCfcgXBSDKO8PRlEAIY5WOAwKLClL+AEYtgISzSFncqx58swzGAwGCaUM8YncFD37LZ/QbXm+y1q/nzfuN1tgYfVUBna8rmnc4vv0dpWFS5yk4mlpC6yYamk5HPyMlrUoz77ffP8/ye+XwRKrAAA=" } }

logger = logging.getLogger()
logger.setLevel(logging.INFO)

s3BucketName = 'samplebucket'
s3KeyPrefix = 'cloudwatchlogs/'

def writeit(theBucket, theS3Key, fileContent):
  try:
    client = boto3.client('s3')
    response = client.put_object(
        ACL='bucket-owner-full-control',
        # Body=b'bytes'|file,
        Body=str(fileContent),
        Bucket=theBucket,
        #CacheControl='string',
        #ContentDisposition='string',
        #ContentEncoding='string',
        #ContentLanguage='string',
        #ContentLength=123,
        #ContentMD5='string',
        #ContentType='string',
        #Expires=datetime(2015, 1, 1),
        #GrantFullControl='string',
        #GrantRead='string',
        #GrantReadACP='string',
        #GrantWriteACP='string',
        Key=theS3Key,
        #Metadata={
        #    'string': 'string'
        #},
        ServerSideEncryption='AES256',
        StorageClass='STANDARD'
        )
  except ClientError as e:
        print('Write of s3://' + s3BucketName + '/' + s3KeyPrefix + ' failed. Raising exception.')
        print(e)
  except Exception as e:
        print('Write of s3://' + s3BucketName + '/' + s3KeyPrefix + ' failed. Raising exception.')
        print(e)
  return        

def lambda_handler(event, context):
    print(json.dumps(event))
    try:
        s3BucketName = os.environ['s3BucketName']
        s3KeyPrefix = os.environ['s3KeyPrefix']
        #print("Environment variables: " + s3BucketName + '/' + s3KeyPrefix)
    except:
        print("FATAL ERROR. Lambda environment variable 's3BucketName' or 's3KeyPrefix' not set.")
        return "failed"
    print("Using: " + s3BucketName + '/' + s3KeyPrefix)
    #capture the CloudWatch log data
    outEvent = str(event['awslogs']['data'])
    
    #decode and unzip the log data
    #outEvent = gzip.GzipFile(fileobj=StringIO(outEvent.decode('base64','strict'))).read()
    outEvent = gzip.GzipFile(fileobj=BytesIO(base64.b64decode(outEvent))).read()
    
    #convert the log data from JSON into a dictionary
    cleanEvent = json.loads(outEvent)
    #print(str(cleanEvent))
    logGroup = cleanEvent['logGroup']
    logStream = cleanEvent['logStream']
    logEvents = cleanEvent['logEvents']
    fileContent = str(json.dumps(logEvents))
    #print(fileContent)    
        
    tstamp = '{0:.0f}'.format(((datetime.datetime.now() - datetime.datetime(1970, 1, 1)).total_seconds())*100)
    theS3Key = s3KeyPrefix + '/' + logGroup + '/' + logStream + '/' + tstamp
    
    # Write it out again
    writeit(s3BucketName, theS3Key, fileContent)

    #log ending
    print("Ending now...")
    return "success"

# Test harness    
if __name__ == "__main__":
    print("starting main")
    lambda_handler(event, "")
    print("ending main")