AWS Elastic Beanstalk – Script timed out before returning headers: application.py

I have an existing Elastic Beanstalk flask app on AWS that occasionally will not initialize and gives the following error:

[Mon Jan 23 10:06:51.550205 2017] [core:error] [pid 7331] [client  127.0.0.1:43790] script timed out before returning headers: application.py
[Mon Jan 23 10:10:43.910014 2017] [core:error] [pid 7329] [client 127.0.0.1:43782] End of script output before headers: application.py

Any ideas why this might be? Most recently I changed the project’s requirements.txt to include pandas==0.19.2. Prior to that change, the program would work for several days before returning the same error. More logs/program details:

[Mon Jan 23 10:05:36.877664 2017] [suexec:notice] [pid 7323] AH01232: suEXEC mechanism enabled (wrapper: /usr/sbin/suexec)
[Mon Jan 23 10:05:36.886151 2017] [so:warn] [pid 7323] AH01574: module wsgi_module is already loaded, skipping
AH00557: httpd: apr_sockaddr_info_get() failed for ip-10-55-254-33
AH00558: httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1. Set the 'ServerName' directive globally to suppress this message
[Mon Jan 23 10:05:36.887302 2017] [auth_digest:notice] [pid 7323] AH01757: generating secret for digest authentication ...
[Mon Jan 23 10:05:36.887797 2017] [lbmethod_heartbeat:notice] [pid 7323] AH02282: No slotmem from mod_heartmonitor
[Mon Jan 23 10:05:36.887828 2017] [:warn] [pid 7323] mod_wsgi: Compiled for Python/2.7.10.
[Mon Jan 23 10:05:36.887832 2017] [:warn] [pid 7323] mod_wsgi: Runtime using Python/2.7.12.
[Mon Jan 23 10:05:36.889729 2017] [mpm_prefork:notice] [pid 7323] AH00163: Apache/2.4.23 (Amazon) mod_wsgi/3.5 Python/2.7.12 configured -- resuming normal operations
[Mon Jan 23 10:05:36.889744 2017] [core:notice] [pid 7323] AH00094: Command line: '/usr/sbin/httpd -D FOREGROUND'
[Mon Jan 23 10:06:43.542607 2017] [core:error] [pid 7328] [client 127.0.0.1:43786] Script timed out before returning headers: application.py
[Mon Jan 23 10:06:47.548360 2017] [core:error] [pid 7330] [client 127.0.0.1:43788] Script timed out before returning headers: application.py
[Mon Jan 23 10:06:51.550205 2017] [core:error] [pid 7331] [client 127.0.0.1:43790] Script timed out before returning headers: application.py
[Mon Jan 23 10:10:43.910014 2017] [core:error] [pid 7329] [client 127.0.0.1:43782] End of script output before headers: application.py

application.py

import flask
from flask import request, Response
import logging
import json
import JobType1
import JobType2
import sys


application = flask.Flask(__name__)
application.config.from_object('default_config')
application.debug = application.config['FLASK_DEBUG'] in ['true', 'True']`

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


@application.route('/', methods=['GET'])
def index():
  logger.info("The message received was '/', no action taken")
  response = Response("Success", status=200)
  return response


@application.route('/StartJob', methods=['POST'])
def StartJob():
  logger.info("!!start_job message received! This is the start job logger")
  print("!!start_job message received! This is the start job printer")
  response = None

if request.json is None:
    response = Response("Error, no job specified.", status=400)
else:
    message = dict()

    try:
        if request.json.has_key('TopicArn') and request.json.has_key('Message'):
            message = json.loads(request.json['Message'])
            job = message['job']
        else:
            message = request.json
            job = message['job']
        date_key = None
        try:
            date_key = message['runkey']
        except Exception as e:
            print "printing Exception:"
            print e
            pass
        start_job(job, date_key)
        response = Response("Called Job", status=200)
    except Exception as ex:
        logging.exception("There was an error processing this message: %s" % request.json)
        response = Response("Error processing message", status=400)
return response


@application.route('/CronMessage', methods=['POST'])
def cron_message():
  logger.info("!!Cron message received! This is the Cron Start Logger")
  response = None
  logger.info("About to print headers of CRON***")
  job = request.headers.get('X-Aws-Sqsd-Taskname')
  start_job(job, None)
  response = Response("Called Job", status=200)
  return response


def start_job(job_name, date_key):
  logger.info("JOB NAME SUBMITTED IS:")
  logger.info(job_name)
  if job_name == 'JobType1':
      start_job_if_not_running(job_name, JobType1.main, True, date_key)
  if job_name == 'JobType2':
    start_job_if_not_running(job_name, JobType2.main, True, date_key)
  else:
    print "Submitted job nome is invalid, no job started. The invalid submitted job name was %s" % job_name


def start_job_if_not_running(job_name, program_to_execute, uses_date_key_flag, date_key):
  global running_jobs
  logger.info("running_jobs are:")
  logger.info(running_jobs)

  if job_name in running_jobs:
    logger.info("Currently running job " + job_name + ", will not start again.")
    return False
else:
    try:
        running_jobs.append(job_name)
        if uses_date_key_flag:
            logger.info("")
            program_to_execute(date_key)
        else:
            program_to_execute()
    except Exception as e:
        handle_job_end(job_name)
        print "Error in " + job_name
        error_message = str(e) + "-" + str(sys.exc_info()[0])
        print error_message
        EmailUsers.main(subject="Error in " + job_name,
                        message=error_message,
                        message_type='error',
                        date_key=date_key,
                        job_name=job_name,
                        process_name='application.py',
                        notification_group='bp_only')
    handle_job_end(job_name)


def handle_job_end(job_name):
  while job_name in running_jobs:
    running_jobs.remove(job_name)

logger.info("Process Complete")

if __name__ == '__main__':
  application.run(host='0.0.0.0', threaded=True)

Any help is appreciated, I can share more code from the other files as necessary.

In addition, if I navigate to /etc/httpd/conf.d/wsgi.conf, I see:

LoadModule wsgi_module modules/mod_wsgi.so
WSGIPythonHome /opt/python/run/baselinenv
WSGISocketPrefix run/wsgi
WSGIRestrictEmbedded On

<VirtualHost *:80>

Alias /static/ /opt/python/current/app/static/
<Directory /opt/python/current/app/static/>
Order allow,deny
Allow from all
</Directory>


WSGIScriptAlias / /opt/python/current/app/application.py


<Directory /opt/python/current/app/>
  Require all granted
</Directory>

WSGIDaemonProcess wsgi processes=1 threads=15 display-name=%{GROUP} 
  python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python2.7/site-packages:/opt/python/run/venv/lib/python2.7/site-packages user=wsgi group=wsgi 
  home=/opt/python/current/app
WSGIProcessGroup wsgi
</VirtualHost>

LogFormat "%h (%{X-Forwarded-For}i) %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" combined