Tutorial 2- PySpark DataFrames- Part 1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "00166693",
   "metadata": {},
   "source": [
    "#### In this Video We will Cover\n",
    "- PySpark Dataframe \n",
    "- Reading The Dataset\n",
    "- Checking the Datatypes of the Column(Schema)\n",
    "- Selecting Columns And Indexing\n",
    "- Check Describe option similar to Pandas\n",
    "- Adding Columns\n",
    "- Dropping columns\n",
    "- Renaming Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fb37d003",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a1d46445",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark=SparkSession.builder.appName('Dataframe').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "078fccba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://host.docker.internal:4041\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.1.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Dataframe</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x1b14e441588>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b7b4a628",
   "metadata": {},
   "outputs": [],
   "source": [
    "## read the dataset\n",
    "df_pyspark=spark.read.option('header','true').csv('test1.csv',inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "79a95903",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Name: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- Experience: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Check the schema\n",
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8a0d131b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+\n",
      "|     Name|age|Experience|\n",
      "+---------+---+----------+\n",
      "|    Krish| 31|        10|\n",
      "|Sudhanshu| 30|         8|\n",
      "|    Sunny| 29|         4|\n",
      "+---------+---+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark=spark.read.csv('test1.csv',header=True,inferSchema=True)\n",
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "47a5a108",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Name: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- Experience: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Check the schema\n",
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "19da5885",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.dataframe.DataFrame"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_pyspark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e4a3c8eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Name='Krish', age=31, Experience=10),\n",
       " Row(Name='Sudhanshu', age=30, Experience=8),\n",
       " Row(Name='Sunny', age=29, Experience=4)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5523ae24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+\n",
      "|     Name|age|Experience|\n",
      "+---------+---+----------+\n",
      "|    Krish| 31|        10|\n",
      "|Sudhanshu| 30|         8|\n",
      "|    Sunny| 29|         4|\n",
      "+---------+---+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c513816d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------+\n",
      "|     Name|Experience|\n",
      "+---------+----------+\n",
      "|    Krish|        10|\n",
      "|Sudhanshu|         8|\n",
      "|    Sunny|         4|\n",
      "+---------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.select(['Name','Experience']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a8bd20e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Column<'Name'>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark['Name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c3d42722",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Name', 'string'), ('age', 'int'), ('Experience', 'int')]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fa74b18e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----+----+-----------------+\n",
      "|summary| Name| age|       Experience|\n",
      "+-------+-----+----+-----------------+\n",
      "|  count|    3|   3|                3|\n",
      "|   mean| null|30.0|7.333333333333333|\n",
      "| stddev| null| 1.0|3.055050463303893|\n",
      "|    min|Krish|  29|                4|\n",
      "|    max|Sunny|  31|               10|\n",
      "+-------+-----+----+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6a8d7b54",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Adding Columns in data frame\n",
    "df_pyspark=df_pyspark.withColumn('Experience After 2 year',df_pyspark['Experience']+2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e9ff01ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+-----------------------+\n",
      "|     Name|age|Experience|Experience After 2 year|\n",
      "+---------+---+----------+-----------------------+\n",
      "|    Krish| 31|        10|                     12|\n",
      "|Sudhanshu| 30|         8|                     10|\n",
      "|    Sunny| 29|         4|                      6|\n",
      "+---------+---+----------+-----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d98641db",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Drop the columns\n",
    "df_pyspark=df_pyspark.drop('Experience After 2 year')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e0a2723f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+\n",
      "|     Name|age|Experience|\n",
      "+---------+---+----------+\n",
      "|    Krish| 31|        10|\n",
      "|Sudhanshu| 30|         8|\n",
      "|    Sunny| 29|         4|\n",
      "+---------+---+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5432faa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+\n",
      "| New Name|age|Experience|\n",
      "+---------+---+----------+\n",
      "|    Krish| 31|        10|\n",
      "|Sudhanshu| 30|         8|\n",
      "|    Sunny| 29|         4|\n",
      "+---------+---+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Rename the columns\n",
    "df_pyspark.withColumnRenamed('Name','New Name').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e5b7d72",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f211a741",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a80b806",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e00f90e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c6cb899",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e0a8c75",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}