How to read text file on AWS Lambda environment

Question

I am trying to read and store the values of a text file in a variable so that I can covert that text file into CSV file afterwards. But I am getting this error that the file is not found. I used the exact path by right clicking on the text file and select copy path. Still getting this error. Any idea on how to fix this
(https://i.stack.imgur.com/5yKzd.jpg)

Here is the Code

import pandas as pd 
import csv 
import json


dataframe1 = pd.read_csv(r'/data.txt', sep="|")

# storing this dataframe in a csv file
dataframe1.to_csv('CSV_CONVERTED.csv', 
                  index = None)


def csv_to_json(event=None, context=None ):
    jsonArray = []
    csvFilePath = r'/CSV_CONVERTED.csv'
    jsonFilePath = r'/data.json'
      
    #read the csv file
    with open(csvFilePath, encoding='utf-8') as csvf: 
        #load csv file data using csv library's dictionary reader
        csvReader = csv.DictReader(csvf) 

        #convert each csv row into python dict
        for row in csvReader: 
            #add this python dict to json array
            jsonArray.append(row)
  
    #convert python jsonArray to JSON String and write to file
    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: 
        jsonString = json.dumps(jsonArray, indent=4)
        jsonf.write(jsonString)
    
    return{
        'statusCode': 200,
        'body': 'Success'
    }

print(csv_to_json())

Here is the error response

{ "errorMessage": "[Errno 2] No such file or directory:
‘/data.txt’", "errorType": "FileNotFoundError", "stackTrace": [
" File "/var/lang/lib/python3.8/imp.py", line 234, in load_modulen return load_source(name, filename, file)n",
" File "/var/lang/lib/python3.8/imp.py", line 171, in load_sourcen module = _load(spec)n",
" File "", line 702, in _loadn",
" File "", line 671, in _load_unlockedn",
" File "", line 843, in exec_modulen",
" File "", line 219, in _call_with_frames_removedn",
" File "/var/task/Convert.py", line 6, in n dataframe1 = pd.read_csv(r’/data.txt’, sep="|")n",
" File "/opt/python/pandas/util/_decorators.py", line 211, in wrappern return func(*args, **kwargs)n",
" File "/opt/python/pandas/util/_decorators.py", line 331, in wrappern return func(*args, **kwargs)n",
" File "/opt/python/pandas/io/parsers/readers.py", line 950, in read_csvn return _read(filepath_or_buffer, kwds)n",
" File "/opt/python/pandas/io/parsers/readers.py", line 605, in _readn parser = TextFileReader(filepath_or_buffer, **kwds)n",
" File "/opt/python/pandas/io/parsers/readers.py", line 1442, in initn self._engine = self._make_engine(f, self.engine)n",
" File "/opt/python/pandas/io/parsers/readers.py", line 1735, in _make_enginen self.handles = get_handle(n",
" File "/opt/python/pandas/io/common.py", line 856, in get_handlen handle = open(n" ] }

Asked By: Afridi Rahman

||

Source

Answer 1

In Lambda, you can only write to /tmp/ directory.
The Lambda deployment package is deployed onto /var/task/ directory.

Hence use /tmp/ directory to write your files to.


data_file_path = f"/var/task/data.txt"
csvFilePath = f"/tmp/CSV_CONVERTED.csv"

dataframe1 = pd.read_csv(data_file_path, sep="|")

# storing this dataframe in a csv file
dataframe1.to_csv(csvFilePath, index = None)


def csv_to_json(event=None, context=None ):
    jsonArray = []
    jsonFilePath = f"/tmp/data.json"
      
    #read the csv file
    with open(csvFilePath, encoding='utf-8') as csvf: 
        #load csv file data using csv library's dictionary reader
        csvReader = csv.DictReader(csvf) 

        #convert each csv row into python dict
        for row in csvReader: 
            #add this python dict to json array
            jsonArray.append(row)
  
    #convert python jsonArray to JSON String and write to file
    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: 
        jsonString = json.dumps(jsonArray, indent=4)
        jsonf.write(jsonString)
    
    return{
        'statusCode': 200,
        'body': 'Success'
    }

print(csv_to_json())

Answered By: Gunesh Shanbhag

How to read text file on AWS Lambda environment

Question:

Answers: