diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb index b9e0ba9e1a4..a56121328c5 100644 --- a/examples/basic/basic.ipynb +++ b/examples/basic/basic.ipynb @@ -28,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -60,9 +60,55 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: feast in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (0.5.0.post0)\n", + "Requirement already satisfied: google in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (2.0.3)\n", + "Requirement already satisfied: tabulate==0.8.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.8.7)\n", + "Requirement already satisfied: pandavro==1.5.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.5.1)\n", + "Requirement already satisfied: pandas==0.* in /home/zzy/.local/lib/python3.7/site-packages (from feast) (0.25.0)\n", + "Requirement already satisfied: google-cloud-core==1.0.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.0.3)\n", + "Requirement already satisfied: grpcio==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.29.0)\n", + "Requirement already satisfied: fastavro<0.23,>=0.22.11 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.22.13)\n", + "Requirement already satisfied: PyYAML==5.1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (5.1.2)\n", + "Requirement already satisfied: googleapis-common-protos==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.51.0)\n", + "Requirement already satisfied: tqdm==4.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (4.46.0)\n", + "Requirement already satisfied: numpy in /home/zzy/.local/lib/python3.7/site-packages (from feast) (1.17.4)\n", + "Requirement already satisfied: confluent-kafka in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.4.2)\n", + "Requirement already satisfied: google-cloud-bigquery-storage==0.7.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.7.0)\n", + "Requirement already satisfied: toml==0.10.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.10.1)\n", + "Requirement already satisfied: protobuf>=3.10 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (3.12.1)\n", + "Requirement already satisfied: google-cloud-bigquery==1.18.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.18.1)\n", + "Requirement already satisfied: pyarrow>=0.15.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.17.1)\n", + "Requirement already satisfied: google-api-core==1.14.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.14.3)\n", + "Requirement already satisfied: google-cloud-storage==1.20.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.20.0)\n", + "Requirement already satisfied: google-auth==1.6.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.6.3)\n", + "Requirement already satisfied: kafka-python==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.4.7)\n", + "Requirement already satisfied: Click==7.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (7.1.2)\n", + "Requirement already satisfied: beautifulsoup4 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google->feast) (4.9.1)\n", + "Requirement already satisfied: six>=1.9 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from pandavro==1.5.*->feast) (1.14.0)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /home/zzy/.local/lib/python3.7/site-packages (from pandas==0.*->feast) (2.8.0)\n", + "Requirement already satisfied: pytz>=2017.2 in /home/zzy/.local/lib/python3.7/site-packages (from pandas==0.*->feast) (2019.1)\n", + "Requirement already satisfied: setuptools in /home/zzy/.local/lib/python3.7/site-packages (from protobuf>=3.10->feast) (41.6.0)\n", + "Requirement already satisfied: google-resumable-media<0.5.0dev,>=0.3.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-cloud-bigquery==1.18.*->feast) (0.4.1)\n", + "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-api-core==1.14.*->feast) (2.23.0)\n", + "Requirement already satisfied: rsa>=3.1.4 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (4.0)\n", + "Requirement already satisfied: cachetools>=2.0.0 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (4.1.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (0.2.8)\n", + "Requirement already satisfied: soupsieve>1.2 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from beautifulsoup4->google->feast) (2.0.1)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (2.9)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (1.25.8)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/zzy/.local/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (2019.6.16)\n", + "Requirement already satisfied: pyasn1>=0.1.3 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from rsa>=3.1.4->google-auth==1.6.*->feast) (0.4.8)\n" + ] + } + ], "source": [ "!pip install feast" ] @@ -76,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -103,45 +149,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_ONLINE_SERVING_URL)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a project workspace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.create_project('customer_project')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the active project" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.set_project('customer_project')" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -158,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -170,9 +184,142 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datetimecustomer_iddaily_transactionstotal_transactions
02020-05-25 00:00:00+00:0010018.70480277
12020-05-25 00:00:00+00:0010027.16388731
22020-05-25 00:00:00+00:0010039.93597668
32020-05-25 00:00:00+00:0010041.10798078
42020-05-25 00:00:00+00:0010058.30738136
52020-05-26 00:00:00+00:0010012.41681114
62020-05-26 00:00:00+00:0010024.8177359
72020-05-26 00:00:00+00:0010034.40971495
82020-05-26 00:00:00+00:0010046.6173176
92020-05-26 00:00:00+00:0010051.03252586
\n", + "
" + ], + "text/plain": [ + " datetime customer_id daily_transactions \\\n", + "0 2020-05-25 00:00:00+00:00 1001 8.704802 \n", + "1 2020-05-25 00:00:00+00:00 1002 7.163887 \n", + "2 2020-05-25 00:00:00+00:00 1003 9.935976 \n", + "3 2020-05-25 00:00:00+00:00 1004 1.107980 \n", + "4 2020-05-25 00:00:00+00:00 1005 8.307381 \n", + "5 2020-05-26 00:00:00+00:00 1001 2.416811 \n", + "6 2020-05-26 00:00:00+00:00 1002 4.817735 \n", + "7 2020-05-26 00:00:00+00:00 1003 4.409714 \n", + "8 2020-05-26 00:00:00+00:00 1004 6.617317 \n", + "9 2020-05-26 00:00:00+00:00 1005 1.032525 \n", + "\n", + " total_transactions \n", + "0 77 \n", + "1 31 \n", + "2 68 \n", + "3 78 \n", + "4 36 \n", + "5 14 \n", + "6 9 \n", + "7 95 \n", + "8 6 \n", + "9 86 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "customer_features = pd.DataFrame(\n", " {\n", @@ -183,7 +330,7 @@ " }\n", ")\n", "\n", - "print(customer_features.head(500))" + "customer_features.head(10)" ] }, { @@ -203,7 +350,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -223,9 +370,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature daily_transactions (ValueType.DOUBLE) added from dataframe.\n", + "Feature total_transactions (ValueType.INT64) added from dataframe.\n", + "\n" + ] + } + ], "source": [ "customer_fs.infer_fields_from_df(customer_features, replace_existing_features=True)" ] @@ -246,19 +403,53 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client.apply(customer_fs)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature set created: \"customer_transactions\"\n", + "{\n", + " \"spec\": {\n", + " \"name\": \"customer_transactions\",\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"customer_id\",\n", + " \"valueType\": \"INT64\"\n", + " }\n", + " ],\n", + " \"features\": [\n", + " {\n", + " \"name\": \"daily_transactions\",\n", + " \"valueType\": \"DOUBLE\"\n", + " },\n", + " {\n", + " \"name\": \"total_transactions\",\n", + " \"valueType\": \"INT64\"\n", + " }\n", + " ],\n", + " \"maxAge\": \"432000s\",\n", + " \"source\": {\n", + " \"type\": \"KAFKA\",\n", + " \"kafkaSourceConfig\": {\n", + " \"bootstrapServers\": \"kafka:9092,localhost:9094\",\n", + " \"topic\": \"feast-features\"\n", + " }\n", + " },\n", + " \"project\": \"default\"\n", + " },\n", + " \"meta\": {\n", + " \"createdTimestamp\": \"2020-05-27T03:58:07Z\",\n", + " \"status\": \"STATUS_PENDING\"\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "client.apply(customer_fs)\n", "customer_fs = client.get_feature_set(\"customer_transactions\")\n", "print(customer_fs)" ] @@ -272,9 +463,52 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for feature set to be ready for ingestion...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 15/15 [00:01<00:00, 13.99rows/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ingestion complete!\n", + "\n", + "Ingestion statistics:\n", + "Success: 15/15\n", + "Removing temporary file(s)...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "'3b988d56-6885-36c6-804e-73ea76b7eae6'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client.ingest(\"customer_transactions\", customer_features)" ] @@ -302,9 +536,37 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "field_values {\n", + " fields {\n", + " key: \"customer_id\"\n", + " value {\n", + " int64_val: 1001\n", + " }\n", + " }\n", + " fields {\n", + " key: \"daily_transactions\"\n", + " value {\n", + " double_val: 2.460333315469021\n", + " }\n", + " }\n", + " fields {\n", + " key: \"total_transactions\"\n", + " value {\n", + " int64_val: 11\n", + " }\n", + " }\n", + "}\n", + "\n" + ] + } + ], "source": [ "online_features = client.get_online_features(\n", " feature_refs=[\n", @@ -355,20 +617,117 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datetimecustomer_id
02020-05-25 00:00:00+00:001001
12020-05-25 00:00:00+00:001002
22020-05-25 00:00:00+00:001003
32020-05-25 00:00:00+00:001004
42020-05-25 00:00:00+00:001005
52020-05-26 00:00:00+00:001001
62020-05-26 00:00:00+00:001002
72020-05-26 00:00:00+00:001003
82020-05-26 00:00:00+00:001004
92020-05-26 00:00:00+00:001005
\n", + "
" + ], + "text/plain": [ + " datetime customer_id\n", + "0 2020-05-25 00:00:00+00:00 1001\n", + "1 2020-05-25 00:00:00+00:00 1002\n", + "2 2020-05-25 00:00:00+00:00 1003\n", + "3 2020-05-25 00:00:00+00:00 1004\n", + "4 2020-05-25 00:00:00+00:00 1005\n", + "5 2020-05-26 00:00:00+00:00 1001\n", + "6 2020-05-26 00:00:00+00:00 1002\n", + "7 2020-05-26 00:00:00+00:00 1003\n", + "8 2020-05-26 00:00:00+00:00 1004\n", + "9 2020-05-26 00:00:00+00:00 1005" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "event_timestamps = [datetime.utcnow().replace(tzinfo=utc) - timedelta(days=randrange(15), hours=randrange(24), minutes=randrange(60)) for day in range(30)]\n", - "\n", "entity_rows = pd.DataFrame(\n", " {\n", - " \"datetime\": event_timestamps,\n", - " \"customer_id\": [customers[idx % len(customers)] for idx in range(len(event_timestamps))],\n", + " \"datetime\": [day for day in days for customer in customers],\n", + " \"customer_id\": [customer for day in days for customer in customers],\n", " }\n", ")\n", "\n", - "print(entity_rows.head(10))" + "entity_rows.head(10)" ] }, { @@ -387,12 +746,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "batch_client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_BATCH_SERVING_URL)\n", - "batch_client.set_project(\"customer_project\")" + "batch_client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_BATCH_SERVING_URL)" ] }, { @@ -404,7 +762,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "scrolled": true }, @@ -412,8 +770,8 @@ "source": [ "job = batch_client.get_batch_features(\n", " feature_refs=[\n", - " f\"customer_project/daily_transactions\", \n", - " f\"customer_project/total_transactions\", \n", + " f\"daily_transactions\", \n", + " f\"total_transactions\", \n", " ],\n", " entity_rows=entity_rows\n", " )" @@ -428,20 +786,108 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_timestampcustomer_iddaily_transactionstotal_transactions
02020-05-26 00:00:00+00:0010012.41681114
12020-05-26 00:00:00+00:0010046.6173176
22020-05-26 00:00:00+00:0010034.40971495
32020-05-26 00:00:00+00:0010051.03252586
42020-05-26 00:00:00+00:0010024.8177359
\n", + "
" + ], + "text/plain": [ + " event_timestamp customer_id daily_transactions \\\n", + "0 2020-05-26 00:00:00+00:00 1001 2.416811 \n", + "1 2020-05-26 00:00:00+00:00 1004 6.617317 \n", + "2 2020-05-26 00:00:00+00:00 1003 4.409714 \n", + "3 2020-05-26 00:00:00+00:00 1005 1.032525 \n", + "4 2020-05-26 00:00:00+00:00 1002 4.817735 \n", + "\n", + " total_transactions \n", + "0 14 \n", + "1 6 \n", + "2 95 \n", + "3 86 \n", + "4 9 " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = job.to_dataframe()\n", - "print(df.head(10))" + "df.head()" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "feast-ml-py374", "language": "python", - "name": "python3" + "name": "feast-ml-py374" }, "language_info": { "codemirror_mode": { @@ -457,5 +903,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb b/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb index e88fe970d54..c29c01efffb 100644 --- a/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb +++ b/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb @@ -6176,8 +6176,7 @@ "source": [ "os.environ['FEAST_CORE_URL'] = 'localhost:6565'\n", "os.environ['FEAST_ONLINE_URL'] = 'localhost:6566'\n", - "os.environ['FEAST_BATCH_URL'] = 'localhost:6567'\n", - "os.environ['FEAST_PROJECT'] = 'default'" + "os.environ['FEAST_BATCH_URL'] = 'localhost:6567'" ] }, { @@ -6195,8 +6194,7 @@ "metadata": {}, "outputs": [], "source": [ - "client = Client(core_url=os.environ['FEAST_CORE_URL'])\n", - "client.set_project(os.environ['FEAST_PROJECT'])" + "client = Client(core_url=os.environ['FEAST_CORE_URL'])" ] }, { @@ -6490,7 +6488,6 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r", " 0%| | 0/7032 [00:00