lobo/wttr
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

syntax clean up

Browse source
This commit is contained in:
Igor Chubin 2018-10-05 20:19:49 +02:00
parent fcc7b79272
commit fb89c4d985
1 changed files with 109 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

193
lib/srv.py
View file

@ -1,13 +1,11 @@
#!/usr/bin/env python #!/usr/bin/env python
# vim: set encoding=utf-8 # vim: set encoding=utf-8
import gevent from gevent.pywsgi import WSGIServer
from gevent.wsgi import WSGIServer
from gevent.queue import Queue
from gevent.monkey import patch_all from gevent.monkey import patch_all
from gevent.subprocess import Popen, PIPE, STDOUT
patch_all() patch_all()
# pylint: disable=wrong-import-position,wrong-import-order
import sys import sys
import logging import logging
import os import os
@ -20,37 +18,44 @@ import json
import geoip2.database import geoip2.database
import jinja2 import jinja2
from flask import Flask, request, render_template, send_from_directory, send_file, make_response from flask import Flask, request, render_template, \
app = Flask(__name__) send_from_directory, send_file, make_response
APP = Flask(__name__)
MYDIR = os.path.abspath( MYDIR = os.path.abspath(
os.path.dirname(os.path.dirname('__file__'))) os.path.dirname(os.path.dirname('__file__')))
sys.path.append("%s/lib/" % MYDIR) sys.path.append("%s/lib/" % MYDIR)
import wttrin_png, parse_query import wttrin_png
import parse_query
from translations import get_message, FULL_TRANSLATION, PARTIAL_TRANSLATION, SUPPORTED_LANGS from translations import get_message, FULL_TRANSLATION, PARTIAL_TRANSLATION, SUPPORTED_LANGS
from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_2, GITHUB_BUTTON_3, GITHUB_BUTTON_FOOTER from buttons import TWITTER_BUTTON, \
GITHUB_BUTTON, GITHUB_BUTTON_2, GITHUB_BUTTON_3, \
GITHUB_BUTTON_FOOTER
from globals import GEOLITE, \ from globals import GEOLITE, \
IP2LCACHE, ALIASES, BLACKLIST, \ IP2LCACHE, ALIASES, BLACKLIST, \
get_help_file, BASH_FUNCTION_FILE, TRANSLATION_FILE, LOG_FILE, TEST_FILE, \ get_help_file, BASH_FUNCTION_FILE, TRANSLATION_FILE, LOG_FILE, \
TEMPLATES, STATIC, \ TEMPLATES, STATIC, \
NOT_FOUND_LOCATION, \ NOT_FOUND_LOCATION, \
MALFORMED_RESPONSE_HTML_PAGE, \ MALFORMED_RESPONSE_HTML_PAGE, \
IATA_CODES_FILE, \ IATA_CODES_FILE, \
log, error, \ log, \
LISTEN_PORT, LISTEN_HOST, PLAIN_TEXT_AGENTS LISTEN_PORT, LISTEN_HOST, PLAIN_TEXT_AGENTS, PLAIN_TEXT_PAGES, \
IP2LOCATION_KEY
from wttr import get_wetter, get_moon from wttr import get_wetter, get_moon
# pylint: enable=wrong-import-position,wrong-import-order
if not os.path.exists(os.path.dirname(LOG_FILE)): if not os.path.exists(os.path.dirname(LOG_FILE)):
os.makedirs(os.path.dirname(LOG_FILE)) os.makedirs(os.path.dirname(LOG_FILE))
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s %(message)s') logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s %(message)s')
my_loader = jinja2.ChoiceLoader([ MY_LOADER = jinja2.ChoiceLoader([
app.jinja_loader, APP.jinja_loader,
jinja2.FileSystemLoader(TEMPLATES), jinja2.FileSystemLoader(TEMPLATES),
]) ])
app.jinja_loader = my_loader APP.jinja_loader = MY_LOADER
class Limits: class Limits:
def __init__(self): def __init__(self):
@ -106,56 +111,69 @@ class Limits:
self.counter[interval] = {} self.counter[interval] = {}
self.last_update[interval] = t_int / self.divisor[interval] self.last_update[interval] = t_int / self.divisor[interval]
limits = Limits() LIMITS = Limits()
def is_ip(ip): def is_ip(ip_addr):
if re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip) is None: """
Check if `ip_addr` looks like an IP Address
"""
if re.match(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip_addr) is None:
return False return False
try: try:
socket.inet_aton(ip) socket.inet_aton(ip_addr)
return True return True
except socket.error: except socket.error:
return False return False
def location_normalize(location): def location_normalize(location):
"""
Normalize location name `location`
"""
#translation_table = dict.fromkeys(map(ord, '!@#$*;'), None) #translation_table = dict.fromkeys(map(ord, '!@#$*;'), None)
def remove_chars( c, s ): def _remove_chars(chars, string):
return ''.join(x for x in s if x not in c ) return ''.join(x for x in string if x not in chars)
location = location.lower().replace('_', ' ').replace('+', ' ').strip() location = location.lower().replace('_', ' ').replace('+', ' ').strip()
if not location.startswith('moon@'): if not location.startswith('moon@'):
location = remove_chars(r'!@#$*;:\\', location) location = _remove_chars(r'!@#$*;:\\', location)
return location return location
def load_aliases(aliases_filename): def load_aliases(aliases_filename):
"""
Load aliases from the aliases file
"""
aliases_db = {} aliases_db = {}
with open(aliases_filename, 'r') as f: with open(aliases_filename, 'r') as f_aliases:
for line in f.readlines(): for line in f_aliases.readlines():
from_, to_ = line.decode('utf-8').split(':', 1) from_, to_ = line.decode('utf-8').split(':', 1)
aliases_db[location_normalize(from_)] = location_normalize(to_) aliases_db[location_normalize(from_)] = location_normalize(to_)
return aliases_db return aliases_db
def load_iata_codes(iata_codes_filename): def load_iata_codes(iata_codes_filename):
with open(iata_codes_filename, 'r') as f: """
Load IATA codes from the IATA codes file
"""
with open(iata_codes_filename, 'r') as f_iata_codes:
result = [] result = []
for line in f.readlines(): for line in f_iata_codes.readlines():
result.append(line.strip()) result.append(line.strip())
return set(result) return set(result)
location_alias = load_aliases(ALIASES) LOCATION_ALIAS = load_aliases(ALIASES)
location_black_list = [x.strip() for x in open(BLACKLIST, 'r').readlines()] LOCATION_BLACK_LIST = [x.strip() for x in open(BLACKLIST, 'r').readlines()]
iata_codes = load_iata_codes(IATA_CODES_FILE) IATA_CODES = load_iata_codes(IATA_CODES_FILE)
print "IATA CODES LOADED: %s" % len(iata_codes) GEOIP_READER = geoip2.database.Reader(GEOLITE)
def location_canonical_name(location): def location_canonical_name(location):
location = location_normalize(location) location = location_normalize(location)
if location in location_alias: if location in LOCATION_ALIAS:
return location_alias[location.lower()] return LOCATION_ALIAS[location.lower()]
return location return location
def ascii_only(string): def ascii_only(string):
try: try:
for i in range(5): for _ in range(5):
string = string.encode('utf-8') string = string.encode('utf-8')
return True return True
except: except:
@ -188,19 +206,23 @@ def ip2location(ip):
return location return location
try: try:
t = requests.get( 'http://api.ip2location.com/?ip=%s&key=%s&package=WS10' % (IP2LOCATION_KEY, ip)).text ip2location_response = requests\
if ';' in t: .get('http://api.ip2location.com/?ip=%s&key=%s&package=WS10' \
location = t.split(';')[3] % (IP2LOCATION_KEY, ip)).text
if ';' in ip2location_response:
location = ip2location_response.split(';')[3]
open(cached, 'w').write(location) open(cached, 'w').write(location)
print "ip2location says: %s" % location print "ip2location says: %s" % location
return location return location
except: except:
pass pass
reader = geoip2.database.Reader(GEOLITE)
def get_location(ip_addr): def get_location(ip_addr):
response = reader.city(ip_addr) """
Return location pair (CITY, COUNTRY) for `ip_addr`
"""
response = GEOIP_READER.city(ip_addr)
country = response.country.iso_code country = response.country.iso_code
city = response.city.name city = response.city.name
@ -219,28 +241,28 @@ def get_location(ip_addr):
return (city or NOT_FOUND_LOCATION), country return (city or NOT_FOUND_LOCATION), country
def parse_accept_language(acceptLanguage): def parse_accept_language(acceptLanguage):
languages = acceptLanguage.split(",") languages = acceptLanguage.split(",")
locale_q_pairs = [] locale_q_pairs = []
for language in languages: for language in languages:
try: try:
if language.split(";")[0] == language: if language.split(";")[0] == language:
# no q => q = 1 # no q => q = 1
locale_q_pairs.append((language.strip(), "1")) locale_q_pairs.append((language.strip(), "1"))
else: else:
locale = language.split(";")[0].strip() locale = language.split(";")[0].strip()
q = language.split(";")[1].split("=")[1] weight = language.split(";")[1].split("=")[1]
locale_q_pairs.append((locale, q)) locale_q_pairs.append((locale, weight))
except: except:
pass pass
return locale_q_pairs return locale_q_pairs
def find_supported_language(accepted_languages): def find_supported_language(accepted_languages):
for p in accepted_languages: for lang_tuple in accepted_languages:
lang = p[0] lang = lang_tuple[0]
if '-' in lang: if '-' in lang:
lang = lang.split('-',1)[0] lang = lang.split('-', 1)[0]
if lang in SUPPORTED_LANGS: if lang in SUPPORTED_LANGS:
return lang return lang
return None return None
@ -255,24 +277,28 @@ def show_help(location, lang):
text = open(BASH_FUNCTION_FILE, 'r').read() text = open(BASH_FUNCTION_FILE, 'r').read()
elif location == ":translation": elif location == ":translation":
text = open(TRANSLATION_FILE, 'r').read() text = open(TRANSLATION_FILE, 'r').read()
text = text.replace('NUMBER_OF_LANGUAGES', str(len(SUPPORTED_LANGS))).replace('SUPPORTED_LANGUAGES', ' '.join(SUPPORTED_LANGS)) text = text\
.replace('NUMBER_OF_LANGUAGES', str(len(SUPPORTED_LANGS)))\
.replace('SUPPORTED_LANGUAGES', ' '.join(SUPPORTED_LANGS))
return text.decode('utf-8') return text.decode('utf-8')
show_help.pages = [':help', ':bash.function', ':translation' ]
@app.route('/files/<path:path>') @APP.route('/files/<path:path>')
def send_static(path): def send_static(path):
"Send any static file located in /files/"
return send_from_directory(STATIC, path) return send_from_directory(STATIC, path)
@app.route('/favicon.ico') @APP.route('/favicon.ico')
def send_favicon(): def send_favicon():
"Send static file favicon.ico"
return send_from_directory(STATIC, 'favicon.ico') return send_from_directory(STATIC, 'favicon.ico')
@app.route('/malformed-response.html') @APP.route('/malformed-response.html')
def send_malformed(): def send_malformed():
"Send static file malformed-response.html"
return send_from_directory(STATIC, 'malformed-response.html') return send_from_directory(STATIC, 'malformed-response.html')
@app.route("/") @APP.route("/")
@app.route("/<string:location>") @APP.route("/<string:location>")
def wttr(location = None): def wttr(location = None):
""" """
Main rendering function, it processes incoming weather queries. Main rendering function, it processes incoming weather queries.
@ -294,21 +320,21 @@ def wttr(location = None):
lang = hostname[:-8] lang = hostname[:-8]
if request.headers.getlist("X-Forwarded-For"): if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0] ip_addr = request.headers.getlist("X-Forwarded-For")[0]
if ip.startswith('::ffff:'): if ip_addr.startswith('::ffff:'):
ip = ip[7:] ip_addr = ip_addr[7:]
else: else:
ip = request.remote_addr ip_addr = request.remote_addr
try: try:
limits.check_ip(ip) LIMITS.check_ip(ip_addr)
except RuntimeError, e: except RuntimeError, e:
return str(e) return str(e)
except Exception, e: except Exception, e:
logging.error("Exception has occured", exc_info=1) logging.error("Exception has occured", exc_info=1)
return "ERROR" return "ERROR"
if location is not None and location.lower() in location_black_list: if location is not None and location.lower() in LOCATION_BLACK_LIST:
return "" return ""
png_filename = None png_filename = None
@ -321,28 +347,26 @@ def wttr(location = None):
if 'lang' in request.args: if 'lang' in request.args:
lang = request.args.get('lang') lang = request.args.get('lang')
if lang is None and 'Accept-Language' in request.headers: if lang is None and 'Accept-Language' in request.headers:
lang = find_supported_language(parse_accept_language(request.headers.get('Accept-Language', ''))) lang = find_supported_language(
parse_accept_language(
request.headers.get('Accept-Language', '')))
user_agent = request.headers.get('User-Agent', '').lower() user_agent = request.headers.get('User-Agent', '').lower()
html_output = not any(agent in user_agent for agent in PLAIN_TEXT_AGENTS) html_output = not any(agent in user_agent for agent in PLAIN_TEXT_AGENTS)
if location in PLAIN_TEXT_PAGES:
if location in show_help.pages:
help_ = show_help(location, lang) help_ = show_help(location, lang)
if html_output: if html_output:
return render_template('index.html', body=help_) return render_template('index.html', body=help_)
else: return help_
return help_
orig_location = location orig_location = location
if request.headers.getlist("X-Forwarded-For"): if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0] ip_addr = request.headers.getlist("X-Forwarded-For")[0]
if ip.startswith('::ffff:'): if ip_addr.startswith('::ffff:'):
ip = ip[7:] ip_addr = ip_addr[7:]
else: else:
ip = request.remote_addr ip_addr = request.remote_addr
try: try:
# if location is starting with ~ # if location is starting with ~
@ -354,7 +378,7 @@ def wttr(location = None):
if location is not None and not ascii_only(location): if location is not None and not ascii_only(location):
location = "~" + location location = "~" + location
if location is not None and location.upper() in iata_codes: if location is not None and location.upper() in IATA_CODES:
location = '~%s' % location location = '~%s' % location
if location is not None and location.startswith('~'): if location is not None and location.startswith('~'):
@ -367,7 +391,7 @@ def wttr(location = None):
else: else:
location = NOT_FOUND_LOCATION #location[1:] location = NOT_FOUND_LOCATION #location[1:]
try: try:
query_source_location = get_location(ip) query_source_location = get_location(ip_addr)
except: except:
query_source_location = NOT_FOUND_LOCATION, None query_source_location = NOT_FOUND_LOCATION, None
@ -407,7 +431,8 @@ def wttr(location = None):
location = location_canonical_name(location) location = location_canonical_name(location)
log("%s %s %s %s %s %s" \ log("%s %s %s %s %s %s" \
% (ip, user_agent, orig_location, location, query.get('use_imperial', False), lang)) % (ip_addr, user_agent, orig_location, location,
query.get('use_imperial', False), lang))
# We are ready to return the answer # We are ready to return the answer
if png_filename: if png_filename:
@ -434,7 +459,7 @@ def wttr(location = None):
else: else:
if country and location != NOT_FOUND_LOCATION: if country and location != NOT_FOUND_LOCATION:
location = "%s, %s" % (location, country) location = "%s, %s" % (location, country)
output = get_wetter(location, ip, output = get_wetter(location, ip_addr,
html=html_output, html=html_output,
lang=lang, lang=lang,
query=query, query=query,
@ -473,5 +498,5 @@ def wttr(location = None):
logging.error("Exception has occured", exc_info=1) logging.error("Exception has occured", exc_info=1)
return "ERROR" return "ERROR"
server = WSGIServer((LISTEN_HOST, LISTEN_PORT), app) SERVER = WSGIServer((LISTEN_HOST, LISTEN_PORT), APP)
server.serve_forever() SERVER.serve_forever()