how to convert a JSON file to a desired dataframe with Python
Question:
I have a Json file in a local folder, here is the .json file’s content:
{
"id": "TD3$-FFA",
"shortCode": "TD3$-FFA",
"dataSet": {
"id": "TD3C",
"shortCode": "TD3C",
"shortDescription": "Dirty Middle East Gulf to China",
"displayGroup": "BDTI",
"datumUnit": "Worldscale",
"datumPrecision": 2,
"data": [
{
"value": 53.59,
"date": "2023-06-09"
},
{
"value": 67.95,
"date": "2023-06-12"
},
{
"value": 73.77,
"date": "2023-06-13"
}
],
"apiIdentifier": "RDSSYGSJBFEV9P2FLSCXGQC3510G2EGE"
},
"datumUnit": "$/mt",
"datumPrecision": 3,
"projectionStartOn": "2010-05-10T00:00:00",
"projectionEndOn": "2023-06-19T00:00:00",
"groupings": [
{
"date": "2023-06-09T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 14.483,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 13.542,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 12.729,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 11.058,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 11.361,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 12.308,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 13.168,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 13.805,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 11.716,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.151,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 12.984,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.41,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.117,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.522,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 11.974,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.63,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.554,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
}
]
}
]
},
{
"date": "2023-06-12T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 16.409,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 14.864,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 14.259,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 12.182,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 12.378,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 13.092,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 14.027,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 14.246,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 12.94,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.677,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 13.124,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.6,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.286,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.688,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 12.103,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.723,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.661,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
}
]
}
]
},
{
"date": "2023-06-13T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 18.077,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 15.922,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 14.388,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 12.258,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 12.478,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 13.327,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 14.12,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 14.599,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 13.042,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.567,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 13.159,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.624,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.276,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.7,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 12.1,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.734,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.675,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
}
]
}
]
}
],
"apiIdentifier": "RPSVJJTJBXBCAF2FAG2PQAVYN4UGQ9LN"
}
What is the easiest way to convert to below dataframe? Thanks
periodType column has either m or q or y as value which means month, quarter and year
date periodType identifier period value validFrom validTo archiveDate
2023-06-09T00:00:00 m TD3$BALMO Jun 23 14.483 2023-06-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 m TD3$CURMON Jun 23 13.542 2023-06-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 m TD3$+1_M Jul 23 12.729 2023-07-01 2023-07-31 2023-06-09
2023-06-09T00:00:00 m TD3$+2_M Aug 23 11.058 2023-08-01 2023-08-31 2023-06-09
2023-06-09T00:00:00 m TD3$+3_M Sep 23 11.361 2023-09-01 2023-09-29 2023-06-09
2023-06-09T00:00:00 m TD3$+4_M Oct 23 12.308 2023-10-01 2023-10-31 2023-06-09
2023-06-09T00:00:00 m TD3$+5_M Nov 23 13.168 2023-11-01 2023-11-30 2023-06-09
2023-06-09T00:00:00 q TD3$CURQ Q2 23 13.805 2023-04-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 q TD3$+1Q Q3 23 11.716 2023-07-01 2023-09-29 2023-06-09
2023-06-09T00:00:00 q TD3$+2Q Q4 23 14.151 2023-10-01 2023-12-22 2023-06-09
2023-06-09T00:00:00 q TD3$+3Q Q1 24 12.984 2024-01-01 2024-03-29 2023-06-09
2023-06-09T00:00:00 q TD3$+4Q Q2 24 11.41 2024-04-01 2024-06-28 2023-06-09
2023-06-09T00:00:00 q TD3$+5Q Q3 24 11.117 2024-07-01 2024-09-30 2023-06-09
2023-06-09T00:00:00 y TD3$+1CAL Cal 24 12.522 2024-01-01 2024-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+2CAL Cal 25 11.974 2025-01-01 2025-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+3CAL Cal 26 11.63 2026-01-01 2026-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+4CAL Cal 27 11.554 2027-01-01 2027-12-24 2023-06-09
2023-06-12T00:00:00 m TD3$BALMO Jun 23 16.409 2023-06-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 m TD3$CURMON Jun 23 14.864 2023-06-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 m TD3$+1_M Jul 23 14.259 2023-07-01 2023-07-31 2023-06-12
2023-06-12T00:00:00 m TD3$+2_M Aug 23 12.182 2023-08-01 2023-08-31 2023-06-12
2023-06-12T00:00:00 m TD3$+3_M Sep 23 12.378 2023-09-01 2023-09-29 2023-06-12
2023-06-12T00:00:00 m TD3$+4_M Oct 23 13.092 2023-10-01 2023-10-31 2023-06-12
2023-06-12T00:00:00 m TD3$+5_M Nov 23 14.027 2023-11-01 2023-11-30 2023-06-12
2023-06-12T00:00:00 q TD3$CURQ Q2 23 14.246 2023-04-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 q TD3$+1Q Q3 23 12.94 2023-07-01 2023-09-29 2023-06-12
2023-06-12T00:00:00 q TD3$+2Q Q4 23 14.677 2023-10-01 2023-12-22 2023-06-12
2023-06-12T00:00:00 q TD3$+3Q Q1 24 13.124 2024-01-01 2024-03-29 2023-06-12
2023-06-12T00:00:00 q TD3$+4Q Q2 24 11.6 2024-04-01 2024-06-28 2023-06-12
2023-06-12T00:00:00 q TD3$+5Q Q3 24 11.286 2024-07-01 2024-09-30 2023-06-12
2023-06-12T00:00:00 y TD3$+1CAL Cal 24 12.688 2024-01-01 2024-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+2CAL Cal 25 12.103 2025-01-01 2025-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+3CAL Cal 26 11.723 2026-01-01 2026-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+4CAL Cal 27 11.661 2027-01-01 2027-12-24 2023-06-12
Thanks so much!
I am trying to be concise but stackoverflow says "It looks like your post is mostly code; please add some more details." so please ignore the below text
Answers:
Use json_normalize:
import json
with open('file.json') as f:
j = json.load(f)
df = pd.json_normalize(j['groupings'],['groups','projections'],
[['groups','periodType'], 'date']).head()
print (df)
identifier period value validFrom validTo nextRolloverDate
0 TD3$BALMO Jun 23 14.483 2023-06-01 2023-06-30 2023-06-30
1 TD3$CURMON Jun 23 13.542 2023-06-01 2023-06-30 2023-06-30
2 TD3$+1_M Jul 23 12.729 2023-07-01 2023-07-31 2023-06-30
3 TD3$+2_M Aug 23 11.058 2023-08-01 2023-08-31 2023-06-30
4 TD3$+3_M Sep 23 11.361 2023-09-01 2023-09-29 2023-06-30
archiveDate groups.periodType date
0 2023-06-09 m 2023-06-09T00:00:00
1 2023-06-09 m 2023-06-09T00:00:00
2 2023-06-09 m 2023-06-09T00:00:00
3 2023-06-09 m 2023-06-09T00:00:00
4 2023-06-09 m 2023-06-09T00:00:00
I have a Json file in a local folder, here is the .json file’s content:
{
"id": "TD3$-FFA",
"shortCode": "TD3$-FFA",
"dataSet": {
"id": "TD3C",
"shortCode": "TD3C",
"shortDescription": "Dirty Middle East Gulf to China",
"displayGroup": "BDTI",
"datumUnit": "Worldscale",
"datumPrecision": 2,
"data": [
{
"value": 53.59,
"date": "2023-06-09"
},
{
"value": 67.95,
"date": "2023-06-12"
},
{
"value": 73.77,
"date": "2023-06-13"
}
],
"apiIdentifier": "RDSSYGSJBFEV9P2FLSCXGQC3510G2EGE"
},
"datumUnit": "$/mt",
"datumPrecision": 3,
"projectionStartOn": "2010-05-10T00:00:00",
"projectionEndOn": "2023-06-19T00:00:00",
"groupings": [
{
"date": "2023-06-09T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 14.483,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 13.542,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 12.729,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 11.058,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 11.361,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 12.308,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 13.168,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 13.805,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 11.716,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.151,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 12.984,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.41,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.117,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-09"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.522,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 11.974,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.63,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.554,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-09"
}
]
}
]
},
{
"date": "2023-06-12T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 16.409,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 14.864,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 14.259,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 12.182,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 12.378,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 13.092,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 14.027,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 14.246,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 12.94,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.677,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 13.124,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.6,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.286,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-12"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.688,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 12.103,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.723,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.661,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-12"
}
]
}
]
},
{
"date": "2023-06-13T00:00:00",
"groups": [
{
"periodType": "m",
"projections": [
{
"identifier": "TD3$BALMO",
"period": "Jun 23",
"value": 18.077,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$CURMON",
"period": "Jun 23",
"value": 15.922,
"validFrom": "2023-06-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+1_M",
"period": "Jul 23",
"value": 14.388,
"validFrom": "2023-07-01",
"validTo": "2023-07-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2_M",
"period": "Aug 23",
"value": 12.258,
"validFrom": "2023-08-01",
"validTo": "2023-08-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3_M",
"period": "Sep 23",
"value": 12.478,
"validFrom": "2023-09-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4_M",
"period": "Oct 23",
"value": 13.327,
"validFrom": "2023-10-01",
"validTo": "2023-10-31",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+5_M",
"period": "Nov 23",
"value": 14.12,
"validFrom": "2023-11-01",
"validTo": "2023-11-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
}
]
},
{
"periodType": "q",
"projections": [
{
"identifier": "TD3$CURQ",
"period": "Q2 23",
"value": 14.599,
"validFrom": "2023-04-01",
"validTo": "2023-06-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+1Q",
"period": "Q3 23",
"value": 13.042,
"validFrom": "2023-07-01",
"validTo": "2023-09-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2Q",
"period": "Q4 23",
"value": 14.567,
"validFrom": "2023-10-01",
"validTo": "2023-12-22",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3Q",
"period": "Q1 24",
"value": 13.159,
"validFrom": "2024-01-01",
"validTo": "2024-03-29",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4Q",
"period": "Q2 24",
"value": 11.624,
"validFrom": "2024-04-01",
"validTo": "2024-06-28",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+5Q",
"period": "Q3 24",
"value": 11.276,
"validFrom": "2024-07-01",
"validTo": "2024-09-30",
"nextRolloverDate": "2023-06-30",
"archiveDate": "2023-06-13"
}
]
},
{
"periodType": "y",
"projections": [
{
"identifier": "TD3$+1CAL",
"period": "Cal 24",
"value": 12.7,
"validFrom": "2024-01-01",
"validTo": "2024-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+2CAL",
"period": "Cal 25",
"value": 12.1,
"validFrom": "2025-01-01",
"validTo": "2025-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+3CAL",
"period": "Cal 26",
"value": 11.734,
"validFrom": "2026-01-01",
"validTo": "2026-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
},
{
"identifier": "TD3$+4CAL",
"period": "Cal 27",
"value": 11.675,
"validFrom": "2027-01-01",
"validTo": "2027-12-24",
"nextRolloverDate": "2023-12-22",
"archiveDate": "2023-06-13"
}
]
}
]
}
],
"apiIdentifier": "RPSVJJTJBXBCAF2FAG2PQAVYN4UGQ9LN"
}
What is the easiest way to convert to below dataframe? Thanks
periodType column has either m or q or y as value which means month, quarter and year
date periodType identifier period value validFrom validTo archiveDate
2023-06-09T00:00:00 m TD3$BALMO Jun 23 14.483 2023-06-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 m TD3$CURMON Jun 23 13.542 2023-06-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 m TD3$+1_M Jul 23 12.729 2023-07-01 2023-07-31 2023-06-09
2023-06-09T00:00:00 m TD3$+2_M Aug 23 11.058 2023-08-01 2023-08-31 2023-06-09
2023-06-09T00:00:00 m TD3$+3_M Sep 23 11.361 2023-09-01 2023-09-29 2023-06-09
2023-06-09T00:00:00 m TD3$+4_M Oct 23 12.308 2023-10-01 2023-10-31 2023-06-09
2023-06-09T00:00:00 m TD3$+5_M Nov 23 13.168 2023-11-01 2023-11-30 2023-06-09
2023-06-09T00:00:00 q TD3$CURQ Q2 23 13.805 2023-04-01 2023-06-30 2023-06-09
2023-06-09T00:00:00 q TD3$+1Q Q3 23 11.716 2023-07-01 2023-09-29 2023-06-09
2023-06-09T00:00:00 q TD3$+2Q Q4 23 14.151 2023-10-01 2023-12-22 2023-06-09
2023-06-09T00:00:00 q TD3$+3Q Q1 24 12.984 2024-01-01 2024-03-29 2023-06-09
2023-06-09T00:00:00 q TD3$+4Q Q2 24 11.41 2024-04-01 2024-06-28 2023-06-09
2023-06-09T00:00:00 q TD3$+5Q Q3 24 11.117 2024-07-01 2024-09-30 2023-06-09
2023-06-09T00:00:00 y TD3$+1CAL Cal 24 12.522 2024-01-01 2024-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+2CAL Cal 25 11.974 2025-01-01 2025-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+3CAL Cal 26 11.63 2026-01-01 2026-12-24 2023-06-09
2023-06-09T00:00:00 y TD3$+4CAL Cal 27 11.554 2027-01-01 2027-12-24 2023-06-09
2023-06-12T00:00:00 m TD3$BALMO Jun 23 16.409 2023-06-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 m TD3$CURMON Jun 23 14.864 2023-06-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 m TD3$+1_M Jul 23 14.259 2023-07-01 2023-07-31 2023-06-12
2023-06-12T00:00:00 m TD3$+2_M Aug 23 12.182 2023-08-01 2023-08-31 2023-06-12
2023-06-12T00:00:00 m TD3$+3_M Sep 23 12.378 2023-09-01 2023-09-29 2023-06-12
2023-06-12T00:00:00 m TD3$+4_M Oct 23 13.092 2023-10-01 2023-10-31 2023-06-12
2023-06-12T00:00:00 m TD3$+5_M Nov 23 14.027 2023-11-01 2023-11-30 2023-06-12
2023-06-12T00:00:00 q TD3$CURQ Q2 23 14.246 2023-04-01 2023-06-30 2023-06-12
2023-06-12T00:00:00 q TD3$+1Q Q3 23 12.94 2023-07-01 2023-09-29 2023-06-12
2023-06-12T00:00:00 q TD3$+2Q Q4 23 14.677 2023-10-01 2023-12-22 2023-06-12
2023-06-12T00:00:00 q TD3$+3Q Q1 24 13.124 2024-01-01 2024-03-29 2023-06-12
2023-06-12T00:00:00 q TD3$+4Q Q2 24 11.6 2024-04-01 2024-06-28 2023-06-12
2023-06-12T00:00:00 q TD3$+5Q Q3 24 11.286 2024-07-01 2024-09-30 2023-06-12
2023-06-12T00:00:00 y TD3$+1CAL Cal 24 12.688 2024-01-01 2024-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+2CAL Cal 25 12.103 2025-01-01 2025-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+3CAL Cal 26 11.723 2026-01-01 2026-12-24 2023-06-12
2023-06-12T00:00:00 y TD3$+4CAL Cal 27 11.661 2027-01-01 2027-12-24 2023-06-12
Thanks so much!
I am trying to be concise but stackoverflow says "It looks like your post is mostly code; please add some more details." so please ignore the below text
Use json_normalize:
import json
with open('file.json') as f:
j = json.load(f)
df = pd.json_normalize(j['groupings'],['groups','projections'],
[['groups','periodType'], 'date']).head()
print (df)
identifier period value validFrom validTo nextRolloverDate
0 TD3$BALMO Jun 23 14.483 2023-06-01 2023-06-30 2023-06-30
1 TD3$CURMON Jun 23 13.542 2023-06-01 2023-06-30 2023-06-30
2 TD3$+1_M Jul 23 12.729 2023-07-01 2023-07-31 2023-06-30
3 TD3$+2_M Aug 23 11.058 2023-08-01 2023-08-31 2023-06-30
4 TD3$+3_M Sep 23 11.361 2023-09-01 2023-09-29 2023-06-30
archiveDate groups.periodType date
0 2023-06-09 m 2023-06-09T00:00:00
1 2023-06-09 m 2023-06-09T00:00:00
2 2023-06-09 m 2023-06-09T00:00:00
3 2023-06-09 m 2023-06-09T00:00:00
4 2023-06-09 m 2023-06-09T00:00:00