LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from datetime import date, timedelta\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras import callbacks\n",
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\n",
    "    'train.csv', usecols=[1, 2, 3, 4, 5],\n",
    "    dtype={'onpromotion': bool},\n",
    "    converters={'unit_sales': lambda u: np.log1p(\n",
    "        float(u)) if float(u) > 0 else 0},\n",
    "    parse_dates=[\"date\"],\n",
    "    skiprows=range(1, 66458909)  # 2016-01-01\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\n",
    "    \"test.csv\", usecols=[0, 1, 2, 3, 4],\n",
    "    dtype={'onpromotion': bool},\n",
    "    parse_dates=[\"date\"]  # , date_parser=parser\n",
    ").set_index(\n",
    "    ['store_nbr', 'item_nbr', 'date']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "items = pd.read_csv(\n",
    "    \"items.csv\",\n",
    ").set_index(\"item_nbr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_2017 = df_train.loc[df_train.date>=pd.datetime(2017,1,1)]\n",
    "del df_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "promo_2017_train = df_2017.set_index(\n",
    "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(\n",
    "        level=-1).fillna(False)\n",
    "promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\n",
    "promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\n",
    "promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\n",
    "promo_2017_test = promo_2017_test.reindex(promo_2017_train.index).fillna(False)\n",
    "promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\n",
    "del promo_2017_test, promo_2017_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_2017 = df_2017.set_index(\n",
    "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(\n",
    "        level=-1).fillna(0)\n",
    "df_2017.columns = df_2017.columns.get_level_values(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "items = items.reindex(df_2017.index.get_level_values(1))\n",
    "\n",
    "def get_timespan(df, dt, minus, periods, freq='D'):\n",
    "    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_dataset(t2017, is_train=True):\n",
    "    X = pd.DataFrame({\n",
    "        \"day_1_2017\": get_timespan(df_2017, t2017, 1, 1).values.ravel(),\n",
    "        \"mean_3_2017\": get_timespan(df_2017, t2017, 3, 3).mean(axis=1).values,\n",
    "        \"mean_7_2017\": get_timespan(df_2017, t2017, 7, 7).mean(axis=1).values,\n",
    "        \"mean_14_2017\": get_timespan(df_2017, t2017, 14, 14).mean(axis=1).values,\n",
    "        \"mean_30_2017\": get_timespan(df_2017, t2017, 30, 30).mean(axis=1).values,\n",
    "        \"mean_60_2017\": get_timespan(df_2017, t2017, 60, 60).mean(axis=1).values,\n",
    "        \"mean_140_2017\": get_timespan(df_2017, t2017, 140, 140).mean(axis=1).values,\n",
    "        \"promo_14_2017\": get_timespan(promo_2017, t2017, 14, 14).sum(axis=1).values,\n",
    "        \"promo_60_2017\": get_timespan(promo_2017, t2017, 60, 60).sum(axis=1).values,\n",
    "        \"promo_140_2017\": get_timespan(promo_2017, t2017, 140, 140).sum(axis=1).values\n",
    "    })\n",
    "    for i in range(7):\n",
    "        X['mean_4_dow{}_2017'.format(i)] = get_timespan(df_2017, t2017, 28-i, 4, freq='7D').mean(axis=1).values\n",
    "        X['mean_20_dow{}_2017'.format(i)] = get_timespan(df_2017, t2017, 140-i, 20, freq='7D').mean(axis=1).values\n",
    "    for i in range(16):\n",
    "        X[\"promo_{}\".format(i)] = promo_2017[\n",
    "            t2017 + timedelta(days=i)].values.astype(np.uint8)\n",
    "    if is_train:\n",
    "        y = df_2017[\n",
    "            pd.date_range(t2017, periods=16)\n",
    "        ].values\n",
    "        return X, y\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing dataset...\n"
     ]
    }
   ],
   "source": [
    "print(\"Preparing dataset...\")\n",
    "t2017 = date(2017, 5, 31)\n",
    "X_l, y_l = [], []\n",
    "for i in range(6):\n",
    "    delta = timedelta(days=7 * i)\n",
    "    X_tmp, y_tmp = prepare_dataset(\n",
    "        t2017 + delta\n",
    "    )\n",
    "    X_l.append(X_tmp)\n",
    "    y_l.append(y_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = pd.concat(X_l, axis=0)\n",
    "y_train = np.concatenate(y_l, axis=0)\n",
    "del X_l, y_l\n",
    "X_val, y_val = prepare_dataset(date(2017, 7, 26))\n",
    "X_test = prepare_dataset(date(2017, 8, 16), is_train=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stores_items = pd.DataFrame(index=df_2017.index)\n",
    "test_ids = df_test[['id']]\n",
    "\n",
    "items = items.reindex( stores_items.index.get_level_values(1) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.as_matrix()\n",
    "X_test = X_test.as_matrix()\n",
    "X_val = X_val.as_matrix()\n",
    "X_train = X_train.reshape((X_train.shape[0], 1, X_train.shape[1]))\n",
    "X_test = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]))\n",
    "X_val = X_val.reshape((X_val.shape[0], 1, X_val.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(32, input_shape=(X_train.shape[1],X_train.shape[2])))\n",
    "model.add(Dropout(.1))\n",
    "model.add(Dense(32))\n",
    "model.add(Dropout(.2))\n",
    "model.add(Dense(1))\n",
    "model.compile(loss = 'mse', optimizer='adam', metrics=['mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N_EPOCHS = 5\n",
    "\n",
    "val_pred = []\n",
    "test_pred = []\n",
    "# wtpath = 'weights.hdf5'  # To save best epoch. But need Keras bug to be fixed first.\n",
    "sample_weights=np.array( pd.concat([items[\"perishable\"]] * 6) * 0.25 + 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "Step 1\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 26s - loss: 0.3990 - mean_squared_error: 0.3760 - val_loss: 0.3116 - val_mean_squared_error: 0.3116\n",
      "Epoch 2/5\n",
      " - 24s - loss: 0.3554 - mean_squared_error: 0.3363 - val_loss: 0.3021 - val_mean_squared_error: 0.3021\n",
      "Epoch 3/5\n",
      " - 23s - loss: 0.3457 - mean_squared_error: 0.3275 - val_loss: 0.2972 - val_mean_squared_error: 0.2972\n",
      "Epoch 4/5\n",
      " - 22s - loss: 0.3406 - mean_squared_error: 0.3228 - val_loss: 0.2982 - val_mean_squared_error: 0.2982\n",
      "Epoch 5/5\n",
      " - 23s - loss: 0.3384 - mean_squared_error: 0.3209 - val_loss: 0.2987 - val_mean_squared_error: 0.2987\n",
      "==================================================\n",
      "Step 2\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 25s - loss: 0.3660 - mean_squared_error: 0.3452 - val_loss: 0.3426 - val_mean_squared_error: 0.3426\n",
      "Epoch 2/5\n",
      " - 24s - loss: 0.3619 - mean_squared_error: 0.3415 - val_loss: 0.3266 - val_mean_squared_error: 0.3266\n",
      "Epoch 3/5\n",
      " - 23s - loss: 0.3615 - mean_squared_error: 0.3411 - val_loss: 0.3282 - val_mean_squared_error: 0.3282\n",
      "Epoch 4/5\n",
      " - 20s - loss: 0.3604 - mean_squared_error: 0.3401 - val_loss: 0.3255 - val_mean_squared_error: 0.3255\n",
      "Epoch 5/5\n",
      " - 20s - loss: 0.3601 - mean_squared_error: 0.3398 - val_loss: 0.3306 - val_mean_squared_error: 0.3306\n",
      "==================================================\n",
      "Step 3\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 20s - loss: 0.3706 - mean_squared_error: 0.3508 - val_loss: 0.3421 - val_mean_squared_error: 0.3421\n",
      "Epoch 2/5\n",
      " - 20s - loss: 0.3647 - mean_squared_error: 0.3455 - val_loss: 0.3409 - val_mean_squared_error: 0.3409\n",
      "Epoch 3/5\n",
      " - 21s - loss: 0.3637 - mean_squared_error: 0.3446 - val_loss: 0.3398 - val_mean_squared_error: 0.3398\n",
      "Epoch 4/5\n",
      " - 22s - loss: 0.3629 - mean_squared_error: 0.3438 - val_loss: 0.3398 - val_mean_squared_error: 0.3398\n",
      "Epoch 5/5\n",
      " - 23s - loss: 0.3626 - mean_squared_error: 0.3436 - val_loss: 0.3401 - val_mean_squared_error: 0.3401\n",
      "==================================================\n",
      "Step 4\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 21s - loss: 0.3991 - mean_squared_error: 0.3772 - val_loss: 0.3609 - val_mean_squared_error: 0.3609\n",
      "Epoch 2/5\n",
      " - 21s - loss: 0.3915 - mean_squared_error: 0.3702 - val_loss: 0.3571 - val_mean_squared_error: 0.3571\n",
      "Epoch 3/5\n",
      " - 21s - loss: 0.3903 - mean_squared_error: 0.3692 - val_loss: 0.3552 - val_mean_squared_error: 0.3552\n",
      "Epoch 4/5\n",
      " - 21s - loss: 0.3897 - mean_squared_error: 0.3686 - val_loss: 0.3533 - val_mean_squared_error: 0.3533\n",
      "Epoch 5/5\n",
      " - 21s - loss: 0.3887 - mean_squared_error: 0.3677 - val_loss: 0.3625 - val_mean_squared_error: 0.3625\n",
      "==================================================\n",
      "Step 5\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 21s - loss: 0.4105 - mean_squared_error: 0.3882 - val_loss: 0.3615 - val_mean_squared_error: 0.3615\n",
      "Epoch 2/5\n",
      " - 21s - loss: 0.4024 - mean_squared_error: 0.3808 - val_loss: 0.3599 - val_mean_squared_error: 0.3599\n",
      "Epoch 3/5\n",
      " - 21s - loss: 0.4012 - mean_squared_error: 0.3797 - val_loss: 0.3586 - val_mean_squared_error: 0.3586\n",
      "Epoch 4/5\n",
      " - 21s - loss: 0.3999 - mean_squared_error: 0.3785 - val_loss: 0.3586 - val_mean_squared_error: 0.3586\n",
      "Epoch 5/5\n",
      " - 21s - loss: 0.3997 - mean_squared_error: 0.3783 - val_loss: 0.3661 - val_mean_squared_error: 0.3661\n",
      "==================================================\n",
      "Step 6\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 21s - loss: 0.4004 - mean_squared_error: 0.3775 - val_loss: 0.3600 - val_mean_squared_error: 0.3600\n",
      "Epoch 2/5\n",
      " - 22s - loss: 0.3930 - mean_squared_error: 0.3706 - val_loss: 0.3649 - val_mean_squared_error: 0.3649\n",
      "Epoch 3/5\n",
      " - 22s - loss: 0.3914 - mean_squared_error: 0.3691 - val_loss: 0.3597 - val_mean_squared_error: 0.3597\n",
      "Epoch 4/5\n",
      " - 23s - loss: 0.3908 - mean_squared_error: 0.3686 - val_loss: 0.3621 - val_mean_squared_error: 0.3621\n",
      "Epoch 5/5\n",
      " - 22s - loss: 0.3901 - mean_squared_error: 0.3679 - val_loss: 0.3618 - val_mean_squared_error: 0.3618\n",
      "==================================================\n",
      "Step 7\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 21s - loss: 0.3866 - mean_squared_error: 0.3653 - val_loss: 0.4123 - val_mean_squared_error: 0.4123\n",
      "Epoch 2/5\n",
      " - 22s - loss: 0.3811 - mean_squared_error: 0.3603 - val_loss: 0.4126 - val_mean_squared_error: 0.4126\n",
      "Epoch 3/5\n",
      " - 22s - loss: 0.3803 - mean_squared_error: 0.3596 - val_loss: 0.4129 - val_mean_squared_error: 0.4129\n",
      "Epoch 4/5\n",
      " - 21s - loss: 0.3796 - mean_squared_error: 0.3590 - val_loss: 0.4165 - val_mean_squared_error: 0.4165\n",
      "Epoch 5/5\n",
      " - 21s - loss: 0.3792 - mean_squared_error: 0.3586 - val_loss: 0.4441 - val_mean_squared_error: 0.4441\n",
      "==================================================\n",
      "Step 8\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 22s - loss: 0.3746 - mean_squared_error: 0.3554 - val_loss: 0.4025 - val_mean_squared_error: 0.4025\n",
      "Epoch 2/5\n",
      " - 22s - loss: 0.3676 - mean_squared_error: 0.3492 - val_loss: 0.3960 - val_mean_squared_error: 0.3960\n",
      "Epoch 3/5\n",
      " - 22s - loss: 0.3662 - mean_squared_error: 0.3480 - val_loss: 0.4038 - val_mean_squared_error: 0.4038\n",
      "Epoch 4/5\n",
      " - 23s - loss: 0.3657 - mean_squared_error: 0.3475 - val_loss: 0.4007 - val_mean_squared_error: 0.4007\n",
      "Epoch 5/5\n",
      " - 22s - loss: 0.3649 - mean_squared_error: 0.3468 - val_loss: 0.4181 - val_mean_squared_error: 0.4181\n",
      "==================================================\n",
      "Step 9\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 21s - loss: 0.3815 - mean_squared_error: 0.3603 - val_loss: 0.3829 - val_mean_squared_error: 0.3829\n",
      "Epoch 2/5\n",
      " - 796s - loss: 0.3757 - mean_squared_error: 0.3550 - val_loss: 0.3847 - val_mean_squared_error: 0.3847\n",
      "Epoch 3/5\n",
      " - 15s - loss: 0.3747 - mean_squared_error: 0.3540 - val_loss: 0.3829 - val_mean_squared_error: 0.3829\n",
      "Epoch 4/5\n",
      " - 13s - loss: 0.3740 - mean_squared_error: 0.3534 - val_loss: 0.3962 - val_mean_squared_error: 0.3962\n",
      "Epoch 5/5\n",
      " - 12s - loss: 0.3736 - mean_squared_error: 0.3530 - val_loss: 0.3790 - val_mean_squared_error: 0.3790\n",
      "==================================================\n",
      "Step 10\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 13s - loss: 0.3880 - mean_squared_error: 0.3675 - val_loss: 0.3734 - val_mean_squared_error: 0.3734\n",
      "Epoch 2/5\n",
      " - 14s - loss: 0.3805 - mean_squared_error: 0.3609 - val_loss: 0.3724 - val_mean_squared_error: 0.3724\n",
      "Epoch 3/5\n",
      " - 18s - loss: 0.3790 - mean_squared_error: 0.3595 - val_loss: 0.3732 - val_mean_squared_error: 0.3732\n",
      "Epoch 4/5\n",
      " - 20s - loss: 0.3782 - mean_squared_error: 0.3588 - val_loss: 0.3692 - val_mean_squared_error: 0.3692\n",
      "Epoch 5/5\n",
      " - 20s - loss: 0.3777 - mean_squared_error: 0.3584 - val_loss: 0.3725 - val_mean_squared_error: 0.3725\n",
      "==================================================\n",
      "Step 11\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 19s - loss: 0.4209 - mean_squared_error: 0.3983 - val_loss: 0.3805 - val_mean_squared_error: 0.3805\n",
      "Epoch 2/5\n",
      " - 13s - loss: 0.4123 - mean_squared_error: 0.3904 - val_loss: 0.3801 - val_mean_squared_error: 0.3801\n",
      "Epoch 3/5\n",
      " - 15s - loss: 0.4109 - mean_squared_error: 0.3892 - val_loss: 0.3760 - val_mean_squared_error: 0.3760\n",
      "Epoch 4/5\n",
      " - 13s - loss: 0.4099 - mean_squared_error: 0.3883 - val_loss: 0.3753 - val_mean_squared_error: 0.3753\n",
      "Epoch 5/5\n",
      " - 12s - loss: 0.4096 - mean_squared_error: 0.3880 - val_loss: 0.3774 - val_mean_squared_error: 0.3774\n",
      "==================================================\n",
      "Step 12\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 13s - loss: 0.4327 - mean_squared_error: 0.4099 - val_loss: 0.3895 - val_mean_squared_error: 0.3895\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/5\n",
      " - 13s - loss: 0.4239 - mean_squared_error: 0.4017 - val_loss: 0.3888 - val_mean_squared_error: 0.3888\n",
      "Epoch 3/5\n",
      " - 17s - loss: 0.4219 - mean_squared_error: 0.3999 - val_loss: 0.3895 - val_mean_squared_error: 0.3895\n",
      "Epoch 4/5\n",
      " - 16s - loss: 0.4202 - mean_squared_error: 0.3983 - val_loss: 0.3899 - val_mean_squared_error: 0.3899\n",
      "Epoch 5/5\n",
      " - 17s - loss: 0.4200 - mean_squared_error: 0.3980 - val_loss: 0.3870 - val_mean_squared_error: 0.3870\n",
      "==================================================\n",
      "Step 13\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 17s - loss: 0.4149 - mean_squared_error: 0.3916 - val_loss: 0.3798 - val_mean_squared_error: 0.3798\n",
      "Epoch 2/5\n",
      " - 15s - loss: 0.4067 - mean_squared_error: 0.3840 - val_loss: 0.3788 - val_mean_squared_error: 0.3788\n",
      "Epoch 3/5\n",
      " - 15s - loss: 0.4055 - mean_squared_error: 0.3829 - val_loss: 0.3806 - val_mean_squared_error: 0.3806\n",
      "Epoch 4/5\n",
      " - 16s - loss: 0.4046 - mean_squared_error: 0.3821 - val_loss: 0.3769 - val_mean_squared_error: 0.3769\n",
      "Epoch 5/5\n",
      " - 16s - loss: 0.4039 - mean_squared_error: 0.3815 - val_loss: 0.3776 - val_mean_squared_error: 0.3776\n",
      "==================================================\n",
      "Step 14\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 17s - loss: 0.3999 - mean_squared_error: 0.3781 - val_loss: 0.3660 - val_mean_squared_error: 0.3660\n",
      "Epoch 2/5\n",
      " - 16s - loss: 0.3939 - mean_squared_error: 0.3727 - val_loss: 0.3633 - val_mean_squared_error: 0.3633\n",
      "Epoch 3/5\n",
      " - 15s - loss: 0.3929 - mean_squared_error: 0.3718 - val_loss: 0.3632 - val_mean_squared_error: 0.3632\n",
      "Epoch 4/5\n",
      " - 16s - loss: 0.3924 - mean_squared_error: 0.3713 - val_loss: 0.3638 - val_mean_squared_error: 0.3638\n",
      "Epoch 5/5\n",
      " - 16s - loss: 0.3916 - mean_squared_error: 0.3706 - val_loss: 0.3650 - val_mean_squared_error: 0.3650\n",
      "==================================================\n",
      "Step 15\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 16s - loss: 0.3891 - mean_squared_error: 0.3694 - val_loss: 0.3537 - val_mean_squared_error: 0.3537\n",
      "Epoch 2/5\n",
      " - 18s - loss: 0.3813 - mean_squared_error: 0.3624 - val_loss: 0.3516 - val_mean_squared_error: 0.3516\n",
      "Epoch 3/5\n",
      " - 18s - loss: 0.3801 - mean_squared_error: 0.3613 - val_loss: 0.3521 - val_mean_squared_error: 0.3521\n",
      "Epoch 4/5\n",
      " - 18s - loss: 0.3794 - mean_squared_error: 0.3607 - val_loss: 0.3501 - val_mean_squared_error: 0.3501\n",
      "Epoch 5/5\n",
      " - 18s - loss: 0.3789 - mean_squared_error: 0.3602 - val_loss: 0.3532 - val_mean_squared_error: 0.3532\n",
      "==================================================\n",
      "Step 16\n",
      "==================================================\n",
      "Train on 1005090 samples, validate on 167515 samples\n",
      "Epoch 1/5\n",
      " - 17s - loss: 0.3970 - mean_squared_error: 0.3752 - val_loss: 0.3757 - val_mean_squared_error: 0.3757\n",
      "Epoch 2/5\n",
      " - 19s - loss: 0.3907 - mean_squared_error: 0.3693 - val_loss: 0.3730 - val_mean_squared_error: 0.3730\n",
      "Epoch 3/5\n",
      " - 19s - loss: 0.3896 - mean_squared_error: 0.3682 - val_loss: 0.3726 - val_mean_squared_error: 0.3726\n",
      "Epoch 4/5\n",
      " - 17s - loss: 0.3893 - mean_squared_error: 0.3680 - val_loss: 0.3723 - val_mean_squared_error: 0.3723\n",
      "Epoch 5/5\n",
      " - 18s - loss: 0.3891 - mean_squared_error: 0.3677 - val_loss: 0.3728 - val_mean_squared_error: 0.3728\n"
     ]
    }
   ],
   "source": [
    "for i in range(16):\n",
    "    print(\"=\" * 50)\n",
    "    print(\"Step %d\" % (i+1))\n",
    "    print(\"=\" * 50)\n",
    "    y = y_train[:, i]\n",
    "    xv = X_val\n",
    "    yv = y_val[:, i]\n",
    "    model.fit(X_train, y, batch_size = 512, epochs = N_EPOCHS, verbose=2,\n",
    "               sample_weight=sample_weights, validation_data=(xv,yv) ) \n",
    "    val_pred.append(model.predict(X_val))\n",
    "    test_pred.append(model.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unweighted validation mse:  0.369163668494\n",
      "Full validation mse:        0.368709261587\n",
      "'Public' validation mse:    0.339996983697\n",
      "'Private' validation mse:   0.381760296992\n"
     ]
    }
   ],
   "source": [
    "n_public = 5 # Number of days in public test set\n",
    "weights=pd.concat([items[\"perishable\"]]) * 0.25 + 1\n",
    "print(\"Unweighted validation mse: \", mean_squared_error(\n",
    "    y_val, np.array(val_pred).squeeze(axis=2).transpose()) )\n",
    "print(\"Full validation mse:       \", mean_squared_error(\n",
    "    y_val, np.array(val_pred).squeeze(axis=2).transpose(), sample_weight=weights) )\n",
    "print(\"'Public' validation mse:   \", mean_squared_error(\n",
    "    y_val[:,:n_public], np.array(val_pred).squeeze(axis=2).transpose()[:,:n_public], \n",
    "    sample_weight=weights) )\n",
    "print(\"'Private' validation mse:  \", mean_squared_error(\n",
    "    y_val[:,n_public:], np.array(val_pred).squeeze(axis=2).transpose()[:,n_public:], \n",
    "    sample_weight=weights) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test = np.array(test_pred).squeeze(axis=2).transpose()\n",
    "df_preds = pd.DataFrame(\n",
    "    y_test, index=stores_items.index,\n",
    "    columns=pd.date_range(\"2017-08-16\", periods=16)\n",
    ").stack().to_frame(\"unit_sales\")\n",
    "df_preds.index.set_names([\"store_nbr\", \"item_nbr\", \"date\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission = test_ids.join(df_preds, how=\"left\").fillna(0)\n",
    "submission[\"unit_sales\"] = np.clip(np.expm1(submission[\"unit_sales\"]), 0, 1000)\n",
    "submission.to_csv('lstm.csv', float_format='%.4f', index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}