Regrouping a list of dictionary

Question

I have a list of dictionary from a database query. I want to group the data by "id" and also add some new key-value pairs based on some existing data in the dictionary.

Here is the data:

data = [{'id': 1, 'name': 'The Musical Hop', 'city': 'San Francisco', 'state': 'CA', 'address': '1015 Folsom Street', 'image_link': 'https://images.unsplash.com/photo-1543900694-133f37abaaa5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=400&q=60', 'facebook_link': 'https://www.facebook.com/TheMusicalHop', 'genres': '{Jazz,Reggae,Swing,Classical,Folk}', 'phone': '123-123-1234', 'website': 'https://www.themusicalhop.com', 'seeking_talent': True, 'seeking_description': 'We are on the lookout for a local artist to play every two weeks. Please call us.', 'artist_id': 4, 'artist_name': 'Gun N Petals', 'artist_image_link': 'https://images.unsplash.com/photo-1549213783-8284d0336c4f?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=300&q=80', 'start_time': datetime.date(2019, 5, 21)}, 
        {'id': 3, 'name': 'Park Square Live Music & Coffee', 'city': 'San Francisco', 'state': 'CA', 'address': '34 Whiskey Moore Ave', 'image_link': 'https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80', 'facebook_link': 'https://www.facebook.com/ParkSquareLiveMusicAndCoffee', 'genres': '{"Rock n Roll",Jazz,Classical,Folk}', 'phone': '415-000-1234', 'website': 'https://www.parksquarelivemusicandcoffee.com', 'seeking_talent': False, 'seeking_description': None, 'artist_id': 5, 'artist_name': 'Matt Quevedo', 'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80', 'start_time': datetime.date(2019, 6, 15)}, 
        {'id': 3, 'name': 'Park Square Live Music & Coffee', 'city': 'San Francisco', 'state': 'CA', 'address': '34 Whiskey Moore Ave', 'image_link': 'https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80', 'facebook_link': 'https://www.facebook.com/ParkSquareLiveMusicAndCoffee', 'genres': '{"Rock n Roll",Jazz,Classical,Folk}', 'phone': '415-000-1234', 'website': 'https://www.parksquarelivemusicandcoffee.com', 'seeking_talent': False, 'seeking_description': None, 'artist_id': 6, 'artist_name': 'The Wild Sax Band', 'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80', 'start_time': datetime.date(2035, 4, 1)}, 
        {'id': 3, 'name': 'Park Square Live Music & Coffee', 'city': 'San Francisco', 'state': 'CA', 'address': '34 Whiskey Moore Ave', 'image_link': 'https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80', 'facebook_link': 'https://www.facebook.com/ParkSquareLiveMusicAndCoffee', 'genres': '{"Rock n Roll",Jazz,Classical,Folk}', 'phone': '415-000-1234', 'website': 'https://www.parksquarelivemusicandcoffee.com', 'seeking_talent': False, 'seeking_description': None, 'artist_id': 6, 'artist_name': 'The Wild Sax Band', 'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80', 'start_time': datetime.date(2035, 4, 8)}, 
        {'id': 3, 'name': 'Park Square Live Music & Coffee', 'city': 'San Francisco', 'state': 'CA', 'address': '34 Whiskey Moore Ave', 'image_link': 'https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80', 'facebook_link': 'https://www.facebook.com/ParkSquareLiveMusicAndCoffee', 'genres': '{"Rock n Roll",Jazz,Classical,Folk}', 'phone': '415-000-1234', 'website': 'https://www.parksquarelivemusicandcoffee.com', 'seeking_talent': False, 'seeking_description': None, 'artist_id': 6, 'artist_name': 'The Wild Sax Band', 'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80', 'start_time': datetime.date(2035, 4, 15)}, 
        {'id': 2, 'name': 'The Dueling Pianos Bar', 'city': 'New York', 'state': 'NY', 'address': '335 Delancey Street', 'image_link': 'https://images.unsplash.com/photo-1497032205916-ac775f0649ae?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=750&q=80', 'facebook_link': 'https://www.facebook.com/theduelingpianos', 'genres': '{Classical,R&B,Hip-Hop}', 'phone': '914-003-1132', 'website': 'https://www.theduelingpianos.com', 'seeking_talent': False, 'seeking_description': None, 'artist_id': None, 'artist_name': None, 'artist_image_link': None, 'start_time': None}]

This is the desired outcome:

data1 = {
        "id": 1,
        "name": "The Musical Hop",
        "genres": ["Jazz", "Reggae", "Swing", "Classical", "Folk"],
        "address": "1015 Folsom Street",
        "city": "San Francisco",
        "state": "CA",
        "phone": "123-123-1234",
        "website": "https://www.themusicalhop.com",
        "facebook_link": "https://www.facebook.com/TheMusicalHop",
        "seeking_talent": True,
        "seeking_description": "We are on the lookout for a local artist to play every two weeks. Please call us.",
        "image_link": "https://images.unsplash.com/photo-1543900694-133f37abaaa5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=400&q=60",
        "past_shows": [{
            "artist_id": 4,
            "artist_name": "Guns N Petals",
            "artist_image_link": "https://images.unsplash.com/photo-1549213783-8284d0336c4f?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=300&q=80",
            "start_time": "2019-05-21T21:30:00.000Z"
        }],
        "upcoming_shows": [],
        "past_shows_count": 1,
        "upcoming_shows_count": 0,
    }
    data2 = {
        "id": 2,
        "name": "The Dueling Pianos Bar",
        "genres": ["Classical", "R&B", "Hip-Hop"],
        "address": "335 Delancey Street",
        "city": "New York",
        "state": "NY",
        "phone": "914-003-1132",
        "website": "https://www.theduelingpianos.com",
        "facebook_link": "https://www.facebook.com/theduelingpianos",
        "seeking_talent": False,
        "image_link": "https://images.unsplash.com/photo-1497032205916-ac775f0649ae?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=750&q=80",
        "past_shows": [],
        "upcoming_shows": [],
        "past_shows_count": 0,
        "upcoming_shows_count": 0,
    }
    data3 = {
        "id": 3,
        "name": "Park Square Live Music & Coffee",
        "genres": ["Rock n Roll", "Jazz", "Classical", "Folk"],
        "address": "34 Whiskey Moore Ave",
        "city": "San Francisco",
        "state": "CA",
        "phone": "415-000-1234",
        "website": "https://www.parksquarelivemusicandcoffee.com",
        "facebook_link": "https://www.facebook.com/ParkSquareLiveMusicAndCoffee",
        "seeking_talent": False,
        "image_link": "https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80",
        "past_shows": [{
            "artist_id": 5,
            "artist_name": "Matt Quevedo",
            "artist_image_link": "https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80",
            "start_time": "2019-06-15T23:00:00.000Z"
        }],
        "upcoming_shows": [{
            "artist_id": 6,
            "artist_name": "The Wild Sax Band",
            "artist_image_link": "https://images.unsplash.com/photo-1558369981-f9ca78462e61?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=794&q=80",
            "start_time": "2035-04-01T20:00:00.000Z"
        }, {
            "artist_id": 6,
            "artist_name": "The Wild Sax Band",
            "artist_image_link": "https://images.unsplash.com/photo-1558369981-f9ca78462e61?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=794&q=80",
            "start_time": "2035-04-08T20:00:00.000Z"
        }, {
            "artist_id": 6,
            "artist_name": "The Wild Sax Band",
            "artist_image_link": "https://images.unsplash.com/photo-1558369981-f9ca78462e61?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=794&q=80",
            "start_time": "2035-04-15T20:00:00.000Z"
        }],
        "past_shows_count": 1,
        "upcoming_shows_count": 3,
    }

Here is what I tried:

venue_list = []

for i, g in itertools.groupby(sorted(data, key=operator.itemgetter("id"), reverse=False, ), 
                              key=operator.itemgetter("start_time")):

    sub_object = list(g)
     
    dates = [{'id': obj['id'], 'start_time': obj['start_time']} for obj in sub_object]
    
    for a, b in itertools.groupby(dates, key=operator.itemgetter("id")):
        sub_object1 = list(b)

        start_date = i
        today = datetime.date.today()

        upcoming_shows = []
        past_shows = []
        past_shows_count = 0
        upcoming_shows_count = 0
        if start_date == None:
            upcoming_shows.append(None)
            past_shows.append(None)

        elif (start_date < today):
            past_shows.append([{'artist_id': obj['artist_id'], 'artist_name': obj['artist_name'], 
                       'artist_image_link': obj['artist_image_link'], 
                           'start_time': start_date} for obj in sub_object])
            past_shows_count = past_shows_count + 1
            upcoming_shows.append(None)

        else:
            upcoming_shows.append([{'artist_id': obj['artist_id'], 'artist_name': obj['artist_name'], 
                       'artist_image_link': obj['artist_image_link'], 
                           'start_time': start_date} for obj in sub_object])
            upcoming_shows_count = upcoming_shows_count + 1
            past_shows.append(None)

    adict = {
        'id': a,
        'name': sub_object[0]['name'],
        'city': sub_object[0]['city'],
        'state': sub_object[0]['state'],
        'address': sub_object[0]['address'],
        'image_link': sub_object[0]['image_link'],
        'facebook_link': sub_object[0]['facebook_link'],
        'genres': sub_object[0]['genres'],
        'phone': sub_object[0]['phone'],
        'website': sub_object[0]['website'],
        'seeking_talent': sub_object[0]['seeking_talent'],
        'seeking_description': sub_object[0]['seeking_description'],
        'past_shows': past_shows,
        'upcoming_shows': upcoming_shows,
        'past_shows_count': past_shows_count,
        'upcoming_shows_count': upcoming_shows_count
    }
    venue_list.append(adict)

NB: upcoming_shows_count and past_shows_count are calculated from count of "start_time".

Asked By: Emmanuel Folorunsho

||

Source

Answer 1

I guess this is what you want, no need to group by twice:

venue_list = []

today = datetime.date.today()
for i, g in itertools.groupby(
        sorted(data, key=operator.itemgetter("id"), reverse=False),
        key=operator.itemgetter("id")
):

    sub_object = list(g)

    upcoming_shows = []
    past_shows = []
    past_shows_count = 0
    upcoming_shows_count = 0
    for obj in sub_object:
        id, start_time = obj['id'], obj['start_time']
        if start_time is None:
            upcoming_shows.append(None)
            past_shows.append(None)

        elif start_time < today:
            past_shows.append(
                {'artist_id': obj['artist_id'], 'artist_name': obj['artist_name'],
                 'artist_image_link': obj['artist_image_link'],
                 'start_time': start_time}
            )
            past_shows_count += 1

        else:
            upcoming_shows.append(
                {'artist_id': obj['artist_id'], 'artist_name': obj['artist_name'],
                 'artist_image_link': obj['artist_image_link'],
                 'start_time': start_time}
            )
            upcoming_shows_count += 1

    adict = {
        **sub_object[0],
        'past_shows': past_shows,
        'upcoming_shows': upcoming_shows,
        'past_shows_count': past_shows_count,
        'upcoming_shows_count': upcoming_shows_count
    }
    venue_list.append(adict)

print(venue_list)

Output:

[
    {
        'id': 1, 'name': 'The Musical Hop', 'city': 'San Francisco', 'state': 'CA', 'address': '1015 Folsom Street',
        'image_link': 'https://images.unsplash.com/photo-1543900694-133f37abaaa5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=400&q=60',
        'facebook_link': 'https://www.facebook.com/TheMusicalHop', 'genres': '{Jazz,Reggae,Swing,Classical,Folk}',
        'phone': '123-123-1234', 'website': 'https://www.themusicalhop.com', 'seeking_talent': True,
        'seeking_description': 'We are on the lookout for a local artist to play every two weeks. Please call us.',
        'artist_id': 4, 'artist_name': 'Gun N Petals',
        'artist_image_link': 'https://images.unsplash.com/photo-1549213783-8284d0336c4f?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=300&q=80',
        'start_time': datetime.date(2019, 5, 21),
        'past_shows': [
            {
                'artist_id': 4,
                'artist_name': 'Gun N Petals',
                'artist_image_link': 'https://images.unsplash.com/photo-1549213783-8284d0336c4f?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=300&q=80',
                'start_time': '2019-05-21'
            }
        ],
        'upcoming_shows': [],
        'past_shows_count': 1,
        'upcoming_shows_count': 0
    },
    {
        'id': 2, 'name': 'The Dueling Pianos Bar', 'city': 'New York', 'state': 'NY', 'address': '335 Delancey Street',
        'image_link': 'https://images.unsplash.com/photo-1497032205916-ac775f0649ae?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=750&q=80',
        'facebook_link': 'https://www.facebook.com/theduelingpianos', 'genres': '{Classical,R&B,Hip-Hop}',
        'phone': '914-003-1132', 'website': 'https://www.theduelingpianos.com', 'seeking_talent': False,
        'seeking_description': None, 'artist_id': None, 'artist_name': None, 'artist_image_link': None,
        'start_time': None,
        'past_shows': [None], 'upcoming_shows': [None], 'past_shows_count': 0, 'upcoming_shows_count': 0},
    {
        'id': 3, 'name': 'Park Square Live Music & Coffee', 'city': 'San Francisco', 'state': 'CA',
        'address': '34 Whiskey Moore Ave',
        'image_link': 'https://images.unsplash.com/photo-1485686531765-ba63b07845a7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=747&q=80',
        'facebook_link': 'https://www.facebook.com/ParkSquareLiveMusicAndCoffee',
        'genres': '{"Rock n Roll",Jazz,Classical,Folk}', 'phone': '415-000-1234',
        'website': 'https://www.parksquarelivemusicandcoffee.com', 'seeking_talent': False, 'seeking_description': None,
        'artist_id': 5, 'artist_name': 'Matt Quevedo',
        'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80',
        'start_time': datetime.date(2019, 6, 15),
        'past_shows': [
            {
                'artist_id': 5,
                'artist_name': 'Matt Quevedo',
                'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80',
                'start_time': '2019-06-15'
            }
        ],
        'upcoming_shows': [
            {
                'artist_id': 6,
                'artist_name': 'The Wild Sax Band',
                'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80',
                'start_time': '2035-04-01'
            },
            {
                'artist_id': 6,
                'artist_name': 'The Wild Sax Band',
                'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80',
                'start_time': '2035-04-08'
            },
            {
                'artist_id': 6,
                'artist_name': 'The Wild Sax Band',
                'artist_image_link': 'https://images.unsplash.com/photo-1495223153807-b916f75de8c5?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&auto=format&fit=crop&w=334&q=80',
                'start_time': '2035-04-15'
            }
        ],
        'past_shows_count': 1,
        'upcoming_shows_count': 3
    }
]

Answered By: funnydman

Regrouping a list of dictionary

Question:

Answers: