airflow/hooks/hive_hooks.py
def load_df(
            self,
            df,
            table,
            field_dict=None,
            delimiter=',',
            encoding='utf8',
            pandas_kwargs=None, **kwargs):
        """
        Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file
        """

        def _infer_field_types_from_df(df):
            DTYPE_KIND_HIVE_TYPE = {
                'b': 'BOOLEAN',    # boolean
                'i': 'BIGINT',     # signed integer
                'u': 'BIGINT',     # unsigned integer
                'f': 'DOUBLE',     # floating-point
                'c': 'STRING',     # complex floating-point
                'M': 'TIMESTAMP',  # datetime
                'O': 'STRING',     # object
                'S': 'STRING',     # (byte-)string
                'U': 'STRING',     # Unicode
                'V': 'STRING'      # void
            }

            d = OrderedDict()
            for col, dtype in df.dtypes.iteritems():
                d[col] = DTYPE_KIND_HIVE_TYPE[dtype.kind]
            return d

        if pandas_kwargs is None:
            pandas_kwargs = {}

        with TemporaryDirectory(prefix='airflow_hiveop_') as tmp_dir:
            with NamedTemporaryFile(dir=tmp_dir, mode="w") as f:

                if field_dict is None:
                    field_dict = _infer_field_types_from_df(df)

                df.to_csv(path_or_buf=f,
                          sep=delimiter,
                          header=False,
                          index=False,
                          encoding=encoding,
                          date_format="%Y-%m-%d %H:%M:%S",
                          **pandas_kwargs)
                f.flush()

                return self.load_file(filepath=f.name,
                                      table=table,
                                      delimiter=delimiter,
                                      field_dict=field_dict,
                                      **kwargs)
def load_df(
            self,
            df,
            table,
            field_dict=None,
            delimiter=',',
            encoding='utf8',
            pandas_kwargs=None, **kwargs):
        """
        Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file
        """

        def _infer_field_types_from_df(df):
            DTYPE_KIND_HIVE_TYPE = {
                'b': 'BOOLEAN',    # boolean
                'i': 'BIGINT',     # signed integer
                'u': 'BIGINT',     # unsigned integer
                'f': 'DOUBLE',     # floating-point
                'c': 'STRING',     # complex floating-point
                'M': 'TIMESTAMP',  # datetime
                'O': 'STRING',     # object
                'S': 'STRING',     # (byte-)string
                'U': 'STRING',     # Unicode
                'V': 'STRING'      # void
            }

            d = OrderedDict()
            for col, dtype in df.dtypes.iteritems():
                d[col] = DTYPE_KIND_HIVE_TYPE[dtype.kind]
            return d

        if pandas_kwargs is None:
            pandas_kwargs = {}

        with TemporaryDirectory(prefix='airflow_hiveop_') as tmp_dir:
            with NamedTemporaryFile(dir=tmp_dir, mode="w") as f:

                if field_dict is None:
                    field_dict = _infer_field_types_from_df(df)

                df.to_csv(path_or_buf=f,
                          sep=delimiter,
                          header=False,
                          index=False,
                          encoding=encoding,
                          date_format="%Y-%m-%d %H:%M:%S",
                          **pandas_kwargs)
                f.flush()

                return self.load_file(filepath=f.name,
                                      table=table,
                                      delimiter=delimiter,
                                      field_dict=field_dict,
                                      **kwargs)
Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file