Read_the_News_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN+EjcDpNm4lW+6LjT8KgwP",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sankytanky100/NLP-Projects/blob/main/Read_the_News_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pre-Processing"
      ],
      "metadata": {
        "id": "PDDnG76Yf1BV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk, re\n",
        "from nltk.corpus import wordnet\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from collections import Counter\n",
        "\n",
        "stop_words = stopwords.words('english')\n",
        "normalizer = WordNetLemmatizer()\n",
        "\n",
        "def get_part_of_speech(word):\n",
        "  probable_part_of_speech = wordnet.synsets(word)\n",
        "  pos_counts = Counter()\n",
        "  pos_counts[\"n\"] = len(  [ item for item in probable_part_of_speech if item.pos()==\"n\"]  )\n",
        "  pos_counts[\"v\"] = len(  [ item for item in probable_part_of_speech if item.pos()==\"v\"]  )\n",
        "  pos_counts[\"a\"] = len(  [ item for item in probable_part_of_speech if item.pos()==\"a\"]  )\n",
        "  pos_counts[\"r\"] = len(  [ item for item in probable_part_of_speech if item.pos()==\"r\"]  )\n",
        "  most_likely_part_of_speech = pos_counts.most_common(1)[0][0]\n",
        "  return most_likely_part_of_speech\n",
        "\n",
        "def preprocess_text(text):\n",
        "  cleaned = re.sub(r'\\W+', ' ', text).lower()\n",
        "  tokenized = word_tokenize(cleaned)\n",
        "  normalized = \" \".join([normalizer.lemmatize(token, get_part_of_speech(token)) for token in tokenized if not re.match(r'\\d+',token)])\n",
        "  return normalized"
      ],
      "metadata": {
        "id": "VSXhIHEsft7s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "article_1 = '''KARACHI: The Sindh government has decided to bring down public transport fares by 7 per cent due to massive reduction in petroleum product prices by the federal government, Geo News reported.Sources said reduction in fares will be applicable on public transport, rickshaw, taxi and other means of traveling. Meanwhile, Karachi Transport Ittehad (KTI) has refused to abide by the government decision.KTI President Irshad Bukhari said the commuters are charged the lowest fares in Karachi as compare to other parts of the country, adding that 80pc vehicles run on Compressed Natural Gas (CNG). Bukhari said Karachi transporters will cut fares when decrease in CNG prices will be made.'''\n",
        "\n",
        "article_2 = '''HONG KONG:  Hong Kong shares opened 0.66 percent lower Monday following a tepid lead from Wall Street, as the first full week of the new year kicked off. The benchmark Hang Seng Index dipped 158.63 points to 23,699.19.'''\n",
        "\n",
        "article_3 = '''KARACHI: Wholesale market rates for sugar dropped to less than Rs 50 per kg following the resumption of sugar cane crushing by sugar mills in Sindh. Within two days, the rate dropped by Rs 1.70 to Rs 49.80 per kg in Karachi Whole Sale Market. According to dealers, the resumption of sugar cane crushing by the mills stabilised the supply to the market with an immediate effect on price as well. Industry experts said that the quality of sugar cane is excellent in Sindh and approximately 100 kg of sugar cane can produce 11 kg of sugar.'''\n",
        "\n",
        "article_4 = '''ISLAMABAD: Long queues of vehicles on fuel stations were visible in different parts of the country as the petrol became rare commodity on Thursday. Federal Minister for Petroleum Shahid Khaqan Abbasi says \"it may take up to ten days to bring the situation to normality\". He claimed that northern areas of Pakistan had been facing the petrol shortage. The minister cited the recent decline in petroleum prices and delay in a shipment as reasons for the shortage.He said situation would improve as soon as shipment reached Pakistan. Sources told Geo News hat due to financial restraints the Pakistan State Oil has been unable import petrol.'''\n",
        "\n",
        "article_5 = '''KARACHI: The final shipment of Chinese manufactured Rail Engines arrived in Pakistan on Friday. Federal Railways Minister, Khwaja Saad Rafique says, the inclusion of the new engines will help ease the shortfall faced by Pakistan Railways. The shipment includes 2000 and 3000-horse-power engines which will be used to pull freight bogeys. Rafique told journalists, the inclusion of 15 new engines has brought Pakistan Railways total strength to 268 engines however more engines are still required.'''\n",
        "\n",
        "article_6 = '''SYDNEY: Cricket fever has gripped Australia with the World Cup just days away. Fans from around the world have thronged to the country and hotels are capitalising. Prices of rooms have almost doubled to 300 dollars and hotels are experiencing full bookings. Experts estimate that during the mega event Australia will generate 1.5 million US dollars just from hotel bookings. If the cost of internal air travel, taxis and tickets is taken into consideration, Australia stands to generate two million US dollars during the World Cup.'''\n",
        "\n",
        "article_7 = '''SAN FRANCISCO: Apple Inc aims to begin producing electric vehicles as early as 2020, Bloomberg reported. The report cited people with knowledge of the matter as saying, a seemingly aggressive target for a mobile devices maker with little experience in car manufacture.The iPhone maker is pushing its \"car team\" of about 200 people to meet that goal. But Apple may decide to scrap its car-making effort, or delay it, if executives grew unhappy with its progress, the news agency said.'''\n",
        "\n",
        "article_8 = '''LAHORE: Federal Minister for Railways, Khawaja Saad Rafique Tuesday announced good news of pay-raise for the employees of Pakistan Railways. In a media statement, the Minister disclosed that a summary for increase in salaries for the employees of Pakistan Railways has been forwarded to the Prime Minister. He also said that the government had also chalked out a plan to build houses for the Railways workers. Khawaja Saad Rafique said it was expected that the salaries of Railway Police may witness a jump of 20 percent. He also announced the government\\x92s plan to launch a new train service between Karachi and Islamabad.'''\n",
        "\n",
        "article_9 = '''ISLAMABAD: The Federal Cabinet on Tuesday approved the budget strategy paper, sources revealed to Geo News. During the cabinet meeting, Prime Minister Nawaz Sharif said tax rate had to be reduced to increase revenue. He added that people would happily pay taxes if the rate was reduced. The prime minister directed the cabinet to provide maximum relief to people in the budget, emphasising that the economic impact should reach people.'''\n",
        "\n",
        "article_10 = '''BEIJING: China will keep the yuan basically stable against a basket of currencies and there is no basis for continued yuan depreciation, central bank vice governor Yi Gang said on Sunday. China also will keep foreign exchange reserves at appropriate levels, Yi said.'''\n",
        "\n",
        "articles = [article_1, article_2, article_3, article_4, article_5, article_6, article_7, article_8, article_9, article_10]"
      ],
      "metadata": {
        "id": "5eMDY6M5gFl3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Enc1a3R3forS"
      },
      "outputs": [],
      "source": [
        "# Step 1: Imports and Data Preparation\n",
        "\n",
        "# Import necessary libraries\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# Articles (replace this with the actual dataset or import it from Kaggle if available)\n",
        "articles = [\n",
        "    \"Article text 1...\",\n",
        "    \"Article text 2...\",\n",
        "    \"Article text 3...\",\n",
        "    \"Article text 4...\",\n",
        "    \"Article text 5...\",\n",
        "    \"Article text 6...\",\n",
        "    \"Article text 7...\",\n",
        "    \"Article text 8...\",\n",
        "    \"Article text 9...\",\n",
        "    \"Article text 10...\",\n",
        "]\n",
        "\n",
        "# Preprocessing function (sample function)\n",
        "def preprocess_text(text):\n",
        "    from nltk.tokenize import word_tokenize\n",
        "    from nltk.corpus import stopwords\n",
        "    from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    tokens = word_tokenize(text.lower())\n",
        "    filtered_tokens = [lemmatizer.lemmatize(word) for word in tokens if word.isalnum() and word not in stop_words]\n",
        "    return \" \".join(filtered_tokens)\n",
        "\n",
        "# Step 2: Preprocess each article\n",
        "processed_articles = [preprocess_text(article) for article in articles]\n",
        "\n",
        "# Step 3: Calculate Tf-idf Scores using CountVectorizer and TfidfTransformer\n",
        "# Initialize CountVectorizer\n",
        "vectorizer = CountVectorizer()\n",
        "\n",
        "# Fit and transform processed articles\n",
        "counts = vectorizer.fit_transform(processed_articles)\n",
        "\n",
        "# Convert word counts to tf-idf scores\n",
        "transformer = TfidfTransformer(norm=None)\n",
        "tfidf_scores_transformed = transformer.fit_transform(counts)\n",
        "\n",
        "# Create a DataFrame for tf-idf scores\n",
        "df_tf_idf_transformed = pd.DataFrame(\n",
        "    tfidf_scores_transformed.todense(),\n",
        "    columns=vectorizer.get_feature_names_out()\n",
        ")\n",
        "print(\"TF-IDF Scores (using CountVectorizer and TfidfTransformer):\")\n",
        "print(df_tf_idf_transformed)\n",
        "\n",
        "# Step 4: Calculate Tf-idf Scores using TfidfVectorizer\n",
        "vectorizer = TfidfVectorizer(norm=None)\n",
        "tfidf_scores = vectorizer.fit_transform(processed_articles)\n",
        "\n",
        "# Create a DataFrame for tf-idf scores\n",
        "df_tf_idf = pd.DataFrame(\n",
        "    tfidf_scores.todense(),\n",
        "    columns=vectorizer.get_feature_names_out()\n",
        ")\n",
        "print(\"TF-IDF Scores (using TfidfVectorizer):\")\n",
        "print(df_tf_idf)\n",
        "\n",
        "# Step 5: Check if the two methods produce the same results\n",
        "if np.allclose(tfidf_scores_transformed.todense(), tfidf_scores.todense()):\n",
        "    print(pd.DataFrame({'Are the tf-idf scores the same?': ['YES']}))\n",
        "else:\n",
        "    print(pd.DataFrame({'Are the tf-idf scores the same?': ['No, something is wrong :(']}))\n",
        "\n",
        "# Step 6: Analyze Results\n",
        "# Find the term with the highest tf-idf score for each article\n",
        "print(\"Top terms by tf-idf score for each article:\")\n",
        "for i in range(1, 11):\n",
        "    print(f\"Article {i}: {df_tf_idf[[f'Article {i-1}']].idxmax()}\")\n"
      ]
    }
  ]
}