sportsref/nba/boxscores.py
def pbp(self, dense_lineups=False, sparse_lineups=False):
        """Returns a dataframe of the play-by-play data from the game.

        :param dense_lineups: If True, adds 10 columns containing the names of
            the players on the court. Defaults to False.
        :param sparse_lineups: If True, adds binary columns denoting whether a
            given player is in the game at the time of a pass. Defaults to
            False.
        :returns: pandas DataFrame of play-by-play. Similar to GPF.
        """
        try:
            doc = self.get_subpage_doc('pbp')
        except:
            raise ValueError(
                'Error fetching PBP subpage for boxscore {}'
                .format(self.boxscore_id)
            )
        table = doc('table#pbp')
        trs = [
            tr for tr in table('tr').items()
            if (not tr.attr['class'] or  # regular data rows
                tr.attr['id'] and tr.attr['id'].startswith('q'))  # qtr bounds
        ]
        rows = [tr.children('td') for tr in trs]
        n_rows = len(trs)
        data = []
        cur_qtr = 0
        bsid = self.boxscore_id

        for i in range(n_rows):
            tr = trs[i]
            row = rows[i]
            p = {}

            # increment cur_qtr when we hit a new quarter
            if tr.attr['id'] and tr.attr['id'].startswith('q'):
                assert int(tr.attr['id'][1:]) == cur_qtr + 1
                cur_qtr += 1
                continue

            # add time of play to entry
            t_str = row.eq(0).text()
            t_regex = r'(\d+):(\d+)\.(\d+)'
            mins, secs, tenths = map(int, re.match(t_regex, t_str).groups())
            endQ = (12 * 60 * min(cur_qtr, 4) +
                    5 * 60 * (cur_qtr - 4 if cur_qtr > 4 else 0))
            secsElapsed = endQ - (60 * mins + secs + 0.1 * tenths)
            p['secs_elapsed'] = secsElapsed
            p['clock_time'] = t_str
            p['quarter'] = cur_qtr

            # handle single play description
            # ex: beginning/end of quarter, jump ball
            if row.length == 2:
                desc = row.eq(1)
                # handle jump balls
                if desc.text().lower().startswith('jump ball: '):
                    p['is_jump_ball'] = True
                    jb_str = sportsref.utils.flatten_links(desc)
                    p.update(
                        sportsref.nba.pbp.parse_play(bsid, jb_str, None)
                    )
                # ignore rows marking beginning/end of quarters
                elif (
                    desc.text().lower().startswith('start of ') or
                    desc.text().lower().startswith('end of ')
                ):
                    continue
                # if another case, log and continue
                else:
                    if not desc.text().lower().startswith('end of '):
                        print(
                            '{}, Q{}, {} other case: {}'
                            .format(self.boxscore_id, cur_qtr,
                                    t_str, desc.text())
                        )
                    continue

            # handle team play description
            # ex: shot, turnover, rebound, foul, sub, etc.
            elif row.length == 6:
                aw_desc, hm_desc = row.eq(1), row.eq(5)
                is_hm_play = bool(hm_desc.text())
                desc = hm_desc if is_hm_play else aw_desc
                desc = sportsref.utils.flatten_links(desc)
                # parse the play
                new_p = sportsref.nba.pbp.parse_play(bsid, desc, is_hm_play)
                if not new_p:
                    continue
                elif isinstance(new_p, list):
                    # this happens when a row needs to be expanded to 2 rows;
                    # ex: double personal foul -> two PF rows

                    # first, update and append the first row
                    orig_p = dict(p)
                    p.update(new_p[0])
                    data.append(p)
                    # second, set up the second row to be appended below
                    p = orig_p
                    new_p = new_p[1]
                elif new_p.get('is_error'):
                    print("can't parse: {}, boxscore: {}"
                          .format(desc, self.boxscore_id))
                    # import pdb; pdb.set_trace()
                p.update(new_p)

            # otherwise, I don't know what this was
            else:
                raise Exception(("don't know how to handle row of length {}"
                                 .format(row.length)))

            data.append(p)

        # convert to DataFrame and clean columns
        df = pd.DataFrame.from_records(data)
        df.sort_values('secs_elapsed', inplace=True, kind='mergesort')
        df = sportsref.nba.pbp.clean_features(df)

        # add columns for home team, away team, boxscore_id, date
        away, home = self.away(), self.home()
        df['home'] = home
        df['away'] = away
        df['boxscore_id'] = self.boxscore_id
        df['season'] = self.season()
        date = self.date()
        df['year'] = date.year
        df['month'] = date.month
        df['day'] = date.day

        def _clean_rebs(df):
            df.reset_index(drop=True, inplace=True)
            no_reb_after = (
                (df.fta_num < df.tot_fta) | df.is_ftm |
                df.get('is_tech_fta', False)
            ).shift(1).fillna(False)
            no_reb_before = (
                (df.fta_num == df.tot_fta)
            ).shift(-1).fillna(False)
            se_end_qtr = df.loc[
                df.clock_time == '0:00.0', 'secs_elapsed'
            ].unique()
            no_reb_when = df.secs_elapsed.isin(se_end_qtr)
            drop_mask = (
                (df.rebounder == 'Team') &
                (no_reb_after | no_reb_before | no_reb_when)
            ).nonzero()[0]
            df.drop(drop_mask, axis=0, inplace=True)
            df.reset_index(drop=True, inplace=True)
            return df

        # get rid of 'rebounds' after FTM, non-final FTA, or tech FTA
        df = _clean_rebs(df)

        # track possession number for each possession
        # TODO: see 201604130PHO, secs_elapsed == 2756
        # things that end a poss:
        # FGM, dreb, TO, end of Q, made last FT, lost jump ball,
        # def goaltending, shot clock violation
        new_poss = (df.off_team == df.home).diff().fillna(False)
        # def rebound considered part of the new possession
        df['poss_id'] = np.cumsum(new_poss) + df.is_dreb
        # create poss_id with rebs -> new possessions for granular groupbys
        poss_id_reb = np.cumsum(new_poss | df.is_reb)

        # make sure plays with the same clock time are in the right order
        # TODO: make sort_cols depend on what cols are in the play?
        # or combine related plays, like and-1 shot and foul
        # issues come up with FGA after timeout in 201604130LAL
        # issues come up with PF between FGA and DREB in 201604120SAS
        sort_cols = [col for col in
                     ['is_reb', 'is_fga', 'is_pf', 'is_tech_foul',
                      'is_ejection', 'is_tech_fta', 'is_timeout', 'is_pf_fta',
                      'fta_num', 'is_viol', 'is_to', 'is_jump_ball', 'is_sub']
                     if col in df.columns]
        asc_true = ['fta_num']
        ascend = [(col in asc_true) for col in sort_cols]
        for label, group in df.groupby([df.secs_elapsed, poss_id_reb]):
            if len(group) > 1:
                df.loc[group.index, :] = group.sort_values(
                    sort_cols, ascending=ascend, kind='mergesort'
                ).values

        # 2nd pass: get rid of 'rebounds' after FTM, non-final FTA, etc.
        df = _clean_rebs(df)

        # makes sure off/def and poss_id are correct for subs after rearranging
        # some possessions above
        df.loc[df['is_sub'], ['off_team', 'def_team', 'poss_id']] = np.nan
        df.off_team.fillna(method='bfill', inplace=True)
        df.def_team.fillna(method='bfill', inplace=True)
        df.poss_id.fillna(method='bfill', inplace=True)
        # make off_team and def_team NaN for jump balls
        if 'is_jump_ball' in df.columns:
            df.loc[df['is_jump_ball'], ['off_team', 'def_team']] = np.nan

        # make sure 'off_team' is always the team shooting FTs, even on techs
        # (impt for keeping track of the score)
        if 'is_tech_fta' in df.columns:
            tech_fta = df['is_tech_fta']
            df.loc[tech_fta, 'off_team'] = df.loc[tech_fta, 'fta_team']
            df.loc[tech_fta, 'def_team'] = np.where(
                df.loc[tech_fta, 'off_team'] == home, away, home
            )
        df.drop('fta_team', axis=1, inplace=True)
        # redefine poss_id_reb
        new_poss = (df.off_team == df.home).diff().fillna(False)
        poss_id_reb = np.cumsum(new_poss | df.is_reb)

        # get rid of redundant subs
        for (se, tm, pnum), group in df[df.is_sub].groupby(
            [df.secs_elapsed, df.sub_team, poss_id_reb]
        ):
            if len(group) > 1:
                sub_in = set()
                sub_out = set()
                # first, figure out who's in and who's out after subs
                for i, row in group.iterrows():
                    if row['sub_in'] in sub_out:
                        sub_out.remove(row['sub_in'])
                    else:
                        sub_in.add(row['sub_in'])
                    if row['sub_out'] in sub_in:
                        sub_in.remove(row['sub_out'])
                    else:
                        sub_out.add(row['sub_out'])
                assert len(sub_in) == len(sub_out)
                # second, add those subs
                n_subs = len(sub_in)
                for idx, p_in, p_out in zip(
                    group.index[:n_subs], sub_in, sub_out
                ):
                    assert df.loc[idx, 'is_sub']
                    df.loc[idx, 'sub_in'] = p_in
                    df.loc[idx, 'sub_out'] = p_out
                    df.loc[idx, 'sub_team'] = tm
                    df.loc[idx, 'detail'] = (
                        '{} enters the game for {}'.format(p_in, p_out)
                    )
                # third, if applicable, remove old sub entries when there are
                # redundant subs
                n_extra = len(group) - len(sub_in)
                if n_extra:
                    extra_idxs = group.index[-n_extra:]
                    df.drop(extra_idxs, axis=0, inplace=True)

        df.reset_index(drop=True, inplace=True)

        # add column for pts and score
        df['pts'] = (df['is_ftm'] + 2 * df['is_fgm'] +
                     (df['is_fgm'] & df['is_three']))
        df['hm_pts'] = np.where(df.off_team == df.home, df.pts, 0)
        df['aw_pts'] = np.where(df.off_team == df.away, df.pts, 0)
        df['hm_score'] = np.cumsum(df['hm_pts'])
        df['aw_score'] = np.cumsum(df['aw_pts'])

        # more helpful columns
        # "play" is differentiated from "poss" by counting OReb as new play
        # "plays" end with non-and1 FGA, TO, last non-tech FTA, or end of qtr
        # (or double lane viol)
        new_qtr = df.quarter.diff().shift(-1).fillna(False).astype(bool)
        and1 = (df.is_fgm & df.is_pf.shift(-1).fillna(False) &
                df.is_fta.shift(-2).fillna(False) &
                ~df.secs_elapsed.diff().shift(-1).fillna(False).astype(bool))
        double_lane = (df.get('viol_type') == 'double lane')
        new_play = df.eval('(is_fga & ~(@and1)) | is_to | @new_qtr |'
                           '(is_fta & ~is_tech_fta & fta_num == tot_fta) |'
                           '@double_lane')
        df['play_id'] = np.cumsum(new_play).shift(1).fillna(0)
        df['hm_off'] = df.off_team == df.home

        # get lineup data
        if dense_lineups:
            df = pd.concat(
                (df, sportsref.nba.pbp.get_dense_lineups(df)), axis=1
            )
        if sparse_lineups:
            df = pd.concat(
                (df, sportsref.nba.pbp.get_sparse_lineups(df)), axis=1
            )

        # TODO: add shot clock as a feature

        return df
def pbp(self, dense_lineups=False, sparse_lineups=False):
        """Returns a dataframe of the play-by-play data from the game.

        :param dense_lineups: If True, adds 10 columns containing the names of
            the players on the court. Defaults to False.
        :param sparse_lineups: If True, adds binary columns denoting whether a
            given player is in the game at the time of a pass. Defaults to
            False.
        :returns: pandas DataFrame of play-by-play. Similar to GPF.
        """
        try:
            doc = self.get_subpage_doc('pbp')
        except:
            raise ValueError(
                'Error fetching PBP subpage for boxscore {}'
                .format(self.boxscore_id)
            )
        table = doc('table#pbp')
        trs = [
            tr for tr in table('tr').items()
            if (not tr.attr['class'] or  # regular data rows
                tr.attr['id'] and tr.attr['id'].startswith('q'))  # qtr bounds
        ]
        rows = [tr.children('td') for tr in trs]
        n_rows = len(trs)
        data = []
        cur_qtr = 0
        bsid = self.boxscore_id

        for i in range(n_rows):
            tr = trs[i]
            row = rows[i]
            p = {}

            # increment cur_qtr when we hit a new quarter
            if tr.attr['id'] and tr.attr['id'].startswith('q'):
                assert int(tr.attr['id'][1:]) == cur_qtr + 1
                cur_qtr += 1
                continue

            # add time of play to entry
            t_str = row.eq(0).text()
            t_regex = r'(\d+):(\d+)\.(\d+)'
            mins, secs, tenths = map(int, re.match(t_regex, t_str).groups())
            endQ = (12 * 60 * min(cur_qtr, 4) +
                    5 * 60 * (cur_qtr - 4 if cur_qtr > 4 else 0))
            secsElapsed = endQ - (60 * mins + secs + 0.1 * tenths)
            p['secs_elapsed'] = secsElapsed
            p['clock_time'] = t_str
            p['quarter'] = cur_qtr

            # handle single play description
            # ex: beginning/end of quarter, jump ball
            if row.length == 2:
                desc = row.eq(1)
                # handle jump balls
                if desc.text().lower().startswith('jump ball: '):
                    p['is_jump_ball'] = True
                    jb_str = sportsref.utils.flatten_links(desc)
                    p.update(
                        sportsref.nba.pbp.parse_play(bsid, jb_str, None)
                    )
                # ignore rows marking beginning/end of quarters
                elif (
                    desc.text().lower().startswith('start of ') or
                    desc.text().lower().startswith('end of ')
                ):
                    continue
                # if another case, log and continue
                else:
                    if not desc.text().lower().startswith('end of '):
                        print(
                            '{}, Q{}, {} other case: {}'
                            .format(self.boxscore_id, cur_qtr,
                                    t_str, desc.text())
                        )
                    continue

            # handle team play description
            # ex: shot, turnover, rebound, foul, sub, etc.
            elif row.length == 6:
                aw_desc, hm_desc = row.eq(1), row.eq(5)
                is_hm_play = bool(hm_desc.text())
                desc = hm_desc if is_hm_play else aw_desc
                desc = sportsref.utils.flatten_links(desc)
                # parse the play
                new_p = sportsref.nba.pbp.parse_play(bsid, desc, is_hm_play)
                if not new_p:
                    continue
                elif isinstance(new_p, list):
                    # this happens when a row needs to be expanded to 2 rows;
                    # ex: double personal foul -> two PF rows

                    # first, update and append the first row
                    orig_p = dict(p)
                    p.update(new_p[0])
                    data.append(p)
                    # second, set up the second row to be appended below
                    p = orig_p
                    new_p = new_p[1]
                elif new_p.get('is_error'):
                    print("can't parse: {}, boxscore: {}"
                          .format(desc, self.boxscore_id))
                    # import pdb; pdb.set_trace()
                p.update(new_p)

            # otherwise, I don't know what this was
            else:
                raise Exception(("don't know how to handle row of length {}"
                                 .format(row.length)))

            data.append(p)

        # convert to DataFrame and clean columns
        df = pd.DataFrame.from_records(data)
        df.sort_values('secs_elapsed', inplace=True, kind='mergesort')
        df = sportsref.nba.pbp.clean_features(df)

        # add columns for home team, away team, boxscore_id, date
        away, home = self.away(), self.home()
        df['home'] = home
        df['away'] = away
        df['boxscore_id'] = self.boxscore_id
        df['season'] = self.season()
        date = self.date()
        df['year'] = date.year
        df['month'] = date.month
        df['day'] = date.day

        def _clean_rebs(df):
            df.reset_index(drop=True, inplace=True)
            no_reb_after = (
                (df.fta_num < df.tot_fta) | df.is_ftm |
                df.get('is_tech_fta', False)
            ).shift(1).fillna(False)
            no_reb_before = (
                (df.fta_num == df.tot_fta)
            ).shift(-1).fillna(False)
            se_end_qtr = df.loc[
                df.clock_time == '0:00.0', 'secs_elapsed'
            ].unique()
            no_reb_when = df.secs_elapsed.isin(se_end_qtr)
            drop_mask = (
                (df.rebounder == 'Team') &
                (no_reb_after | no_reb_before | no_reb_when)
            ).nonzero()[0]
            df.drop(drop_mask, axis=0, inplace=True)
            df.reset_index(drop=True, inplace=True)
            return df

        # get rid of 'rebounds' after FTM, non-final FTA, or tech FTA
        df = _clean_rebs(df)

        # track possession number for each possession
        # TODO: see 201604130PHO, secs_elapsed == 2756
        # things that end a poss:
        # FGM, dreb, TO, end of Q, made last FT, lost jump ball,
        # def goaltending, shot clock violation
        new_poss = (df.off_team == df.home).diff().fillna(False)
        # def rebound considered part of the new possession
        df['poss_id'] = np.cumsum(new_poss) + df.is_dreb
        # create poss_id with rebs -> new possessions for granular groupbys
        poss_id_reb = np.cumsum(new_poss | df.is_reb)

        # make sure plays with the same clock time are in the right order
        # TODO: make sort_cols depend on what cols are in the play?
        # or combine related plays, like and-1 shot and foul
        # issues come up with FGA after timeout in 201604130LAL
        # issues come up with PF between FGA and DREB in 201604120SAS
        sort_cols = [col for col in
                     ['is_reb', 'is_fga', 'is_pf', 'is_tech_foul',
                      'is_ejection', 'is_tech_fta', 'is_timeout', 'is_pf_fta',
                      'fta_num', 'is_viol', 'is_to', 'is_jump_ball', 'is_sub']
                     if col in df.columns]
        asc_true = ['fta_num']
        ascend = [(col in asc_true) for col in sort_cols]
        for label, group in df.groupby([df.secs_elapsed, poss_id_reb]):
            if len(group) > 1:
                df.loc[group.index, :] = group.sort_values(
                    sort_cols, ascending=ascend, kind='mergesort'
                ).values

        # 2nd pass: get rid of 'rebounds' after FTM, non-final FTA, etc.
        df = _clean_rebs(df)

        # makes sure off/def and poss_id are correct for subs after rearranging
        # some possessions above
        df.loc[df['is_sub'], ['off_team', 'def_team', 'poss_id']] = np.nan
        df.off_team.fillna(method='bfill', inplace=True)
        df.def_team.fillna(method='bfill', inplace=True)
        df.poss_id.fillna(method='bfill', inplace=True)
        # make off_team and def_team NaN for jump balls
        if 'is_jump_ball' in df.columns:
            df.loc[df['is_jump_ball'], ['off_team', 'def_team']] = np.nan

        # make sure 'off_team' is always the team shooting FTs, even on techs
        # (impt for keeping track of the score)
        if 'is_tech_fta' in df.columns:
            tech_fta = df['is_tech_fta']
            df.loc[tech_fta, 'off_team'] = df.loc[tech_fta, 'fta_team']
            df.loc[tech_fta, 'def_team'] = np.where(
                df.loc[tech_fta, 'off_team'] == home, away, home
            )
        df.drop('fta_team', axis=1, inplace=True)
        # redefine poss_id_reb
        new_poss = (df.off_team == df.home).diff().fillna(False)
        poss_id_reb = np.cumsum(new_poss | df.is_reb)

        # get rid of redundant subs
        for (se, tm, pnum), group in df[df.is_sub].groupby(
            [df.secs_elapsed, df.sub_team, poss_id_reb]
        ):
            if len(group) > 1:
                sub_in = set()
                sub_out = set()
                # first, figure out who's in and who's out after subs
                for i, row in group.iterrows():
                    if row['sub_in'] in sub_out:
                        sub_out.remove(row['sub_in'])
                    else:
                        sub_in.add(row['sub_in'])
                    if row['sub_out'] in sub_in:
                        sub_in.remove(row['sub_out'])
                    else:
                        sub_out.add(row['sub_out'])
                assert len(sub_in) == len(sub_out)
                # second, add those subs
                n_subs = len(sub_in)
                for idx, p_in, p_out in zip(
                    group.index[:n_subs], sub_in, sub_out
                ):
                    assert df.loc[idx, 'is_sub']
                    df.loc[idx, 'sub_in'] = p_in
                    df.loc[idx, 'sub_out'] = p_out
                    df.loc[idx, 'sub_team'] = tm
                    df.loc[idx, 'detail'] = (
                        '{} enters the game for {}'.format(p_in, p_out)
                    )
                # third, if applicable, remove old sub entries when there are
                # redundant subs
                n_extra = len(group) - len(sub_in)
                if n_extra:
                    extra_idxs = group.index[-n_extra:]
                    df.drop(extra_idxs, axis=0, inplace=True)

        df.reset_index(drop=True, inplace=True)

        # add column for pts and score
        df['pts'] = (df['is_ftm'] + 2 * df['is_fgm'] +
                     (df['is_fgm'] & df['is_three']))
        df['hm_pts'] = np.where(df.off_team == df.home, df.pts, 0)
        df['aw_pts'] = np.where(df.off_team == df.away, df.pts, 0)
        df['hm_score'] = np.cumsum(df['hm_pts'])
        df['aw_score'] = np.cumsum(df['aw_pts'])

        # more helpful columns
        # "play" is differentiated from "poss" by counting OReb as new play
        # "plays" end with non-and1 FGA, TO, last non-tech FTA, or end of qtr
        # (or double lane viol)
        new_qtr = df.quarter.diff().shift(-1).fillna(False).astype(bool)
        and1 = (df.is_fgm & df.is_pf.shift(-1).fillna(False) &
                df.is_fta.shift(-2).fillna(False) &
                ~df.secs_elapsed.diff().shift(-1).fillna(False).astype(bool))
        double_lane = (df.get('viol_type') == 'double lane')
        new_play = df.eval('(is_fga & ~(@and1)) | is_to | @new_qtr |'
                           '(is_fta & ~is_tech_fta & fta_num == tot_fta) |'
                           '@double_lane')
        df['play_id'] = np.cumsum(new_play).shift(1).fillna(0)
        df['hm_off'] = df.off_team == df.home

        # get lineup data
        if dense_lineups:
            df = pd.concat(
                (df, sportsref.nba.pbp.get_dense_lineups(df)), axis=1
            )
        if sparse_lineups:
            df = pd.concat(
                (df, sportsref.nba.pbp.get_sparse_lineups(df)), axis=1
            )

        # TODO: add shot clock as a feature

        return df
Returns a dataframe of the play-by-play data from the game.

        :param dense_lineups: If True, adds 10 columns containing the names of
            the players on the court. Defaults to False.
        :param sparse_lineups: If True, adds binary columns denoting whether a
            given player is in the game at the time of a pass. Defaults to
            False.
        :returns: pandas DataFrame of play-by-play. Similar to GPF.