# Imports & Settings

# Make sure packages in my virtual environment are included
import sys
sys.path.append('/home/hoanvu/anaconda2/envs/ds/lib/python2.7/site-packages/')

import requests
import pickle
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from bs4 import BeautifulSoup
from collections import Counter, defaultdict
from datetime import datetime

%pylab inline
sns.set_context('poster')

# No of URLs to crawl from Vietnamworks
MAX_PAGE = 27

def get_all_urls():
    """
    Scan through all URLs defined by MAX_PAGE and return the URL
    
    return: list of URLs
    """
    
    base_url = 'http://www.vietnamworks.com/it-hardware-networking-it-software-jobs-i55,35-en'
    extended_url_pages = ['/page-{}'.format(page_number) for page_number in xrange(2, MAX_PAGE)]
    urls = [base_url + extended_page for extended_page in extended_url_pages]
    urls.insert(0, base_url)
    return urls

def get_jobs_in_url(url):
    """
    Take a single URL and return a pandas DataFrame contains details about all jobs in that URL.
    
    return: pandas.DataFrame
    """
    req = requests.get(url)
    soup = BeautifulSoup(req.content, 'html.parser')
    
    jobs_table = soup.find('table', {'class': 'link-list'})
    tr_tags = jobs_table.find_all('tr', {'class': 'job-post'})
    
    job_title = []
    job_company = []
    job_location = []
    job_position = []
    job_skills = []
    job_date = []
    
    for job in tr_tags:
        job_title.append(job.find('a', {'class': 'job-title'}).contents[0])
        job_company.append(job.find('span', 'name').contents[0])
        job_location.append(job.find('p', {'class': 'job-info'}).contents[1].find('span').contents[0])
        job_position.append(job.find('p', {'class': 'job-info'}).contents[1].find_all('span')[1].contents[0])
                            
        skill_list_data = job.find('div', {'class': 'skills'})
        if skill_list_data:
            skill_list = skill_list_data.find_all('em', {'class': 'text-clip'})
            
        job_skills.append([skill.contents[0] for skill in skill_list])
        
        # Job post date
        d = job.find('span', {'class': 'views'}).find('span').contents[0].split(' ')[1]
        if d == 'Today':
            d = datetime.datetime.now().strftime("%d/%m/%Y")
        job_date.append(d)
    
        # Job salary
        
    return pd.DataFrame({
            'job_title': job_title,
            'company': job_company,
            'location': job_location,
            'position': job_position,
            'skills': job_skills,
            'post_date': job_date
        })

def get_all_jobs():
    """
    Scans all URLs and crawls all jobs from all pages
    
    return: pandas.DataFrame contains every possible job that we can find
    """
    urls = get_all_urls()
    data = pd.DataFrame()
    for url in urls:
        data = data.append(get_jobs_in_url(url), ignore_index=True)
        
    return data

jobs = get_all_jobs()

# Convert the job post date to datetime type
jobs['post_date'] = pd.to_datetime(jobs.post_date, format="%d/%m/%Y")

jobs.head()

print jobs.post_date.max() - jobs.post_date.min()

print "Total jobs: {}".format(len(jobs))

all_locations = []
for location in jobs.location:
    if ',' in location:
        all_locations.extend(location.split(', '))
    else:
        all_locations.append(location)

# Remove cities that have less than 5 jobs, just to make grap clearer
most_cities = [key for key, value in Counter(all_locations).iteritems() if value > 5]
new_cities = [city for city in all_locations if city in most_cities]

sns.countplot(y=new_cities)

from __future__ import division

location_counter = Counter(all_locations)
total_jobs = sum(location_counter.values())
jobs_in_hcm = location_counter['Ho Chi Minh']
jobs_in_hn = location_counter['Ha Noi']
jobs_in_dn = location_counter['Da Nang']
jobs_in_int = location_counter['International']

print "Total jobs:                     {}".format(total_jobs)
print "Jobs required by Ho Chi Minh:   {:.3f}%".format((jobs_in_hcm / total_jobs) * 100.0 )
print "Jobs required by Ha Noi:        {:.3f}%".format((jobs_in_hn / total_jobs) * 100.0 ) 
print "Jobs required by Da Nang:       {:.3f}%".format((jobs_in_dn / total_jobs) * 100.0 )
print "Jobs required by International: {:.3f}%".format((jobs_in_int / total_jobs) * 100.0) 

sns.countplot(x='position', data=jobs)

# Need to improve this code

skill_counter = defaultdict(int)
for row in jobs.skills:
    for skill in row:
        skill = skill.lower()
        if 'c++' in skill:
            skill_counter['C++'] += 1
        if 'java' in skill or 'j2ee' in skill or 'servlet' in skill or 'jsp' in skill or 'hibernate' in skill:
            skill_counter['Java'] += 1
        if 'ios' in skill or 'objective-c' in skill or 'objective c' in skill:
            skill_counter['iOS'] += 1
        if 'python' in skill or 'django' in skill or 'flask' in skill:
            skill_counter['Python'] += 1
        if 'html' in skill or 'css' in skill or 'css3' in skill:
            skill_counter['HTML/CSS'] += 1
        if 'android' in skill:
            skill_counter['Android'] += 1
        if 'oracle' in skill or 'sql' in skill or 'sql server' in skill or 'mysql' in skill or \
            'database' in skill or 'postgres' in skill:
            skill_counter['Database'] += 1
        if 'linux' in skill or 'redhat' in skill or 'centos' in skill or 'ubuntu' in skill:
            skill_counter['Linux'] += 1
        if 'cisco' in skill or 'ccna' in skill or 'ccnp' in skill or 'ccie' in skill or \
            'network' in skill or 'routing' in skill or 'switching' in skill:
            skill_counter['Network'] += 1
        if 'c#' in skill or '.net' in skill:
            skill_counter['C#/.Net'] += 1
        if 'javascript' in skill:
            skill_counter['Javascript'] += 1
        if 'php' in skill:
            skill_counter['PHP'] += 1
        if 'scrum' in skill:
            skill_counter['Agile Scrum'] += 1
        if 'brse' in skill or 'bridge' in skill:
            skill_counter['BrSE'] += 1

skills = pd.DataFrame(dict(skill_counter), index=range(1))
skills.iloc[0].sort_values(ascending=False).plot(kind='barh')
# sns.barplot(data=pd.DataFrame(dict(skill_counter), index=range(1)), orient='h')
# sns.barplot?

jobs[jobs.location.str.contains('International')][['company', 'job_title']]