backend/passfinder/recomendations/service/service.py
2023-05-27 23:59:50 +03:00

880 lines
28 KiB
Python

from annoy import AnnoyIndex
from .mapping.mapping import *
from .models.models import *
from passfinder.events.models import Event, Region, Hotel, BasePoint, City, Restaurant
from passfinder.events.api.serializers import (
HotelSerializer,
EventSerializer,
RestaurantSerializer,
ObjectRouteSerializer,
)
from passfinder.recomendations.models import *
from random import choice, sample
from collections import Counter
from passfinder.users.models import User
from collections.abc import Iterable
from django.db.models import Q
from geopy.distance import geodesic as GD
from datetime import timedelta, time, datetime
from gevent.pool import Pool
from python_tsp.exact import solve_tsp_dynamic_programming
import numpy as np
def get_nearest_(instance_model, model_type, mapping, rev_mapping, nearest_n, ml_model):
how_many = len(Event.objects.filter(type=model_type))
index = rev_mapping[instance_model.oid]
nearest = ml_model.get_nns_by_item(index, len(mapping))
res = []
for i in range(how_many):
try:
res.append(Event.objects.get(oid=mapping[nearest[i]]))
except Event.DoesNotExist:
...
if len(res) == nearest_n:
break
return res
def nearest_attraction(attraction, nearest_n):
return get_nearest_(
attraction,
"attraction",
attraction_mapping,
rev_attraction_mapping,
nearest_n,
attracion_model,
)
def nearest_mus(museum, nearest_n):
return get_nearest_(
museum,
"museum",
mus_mapping,
rev_mus_mapping,
nearest_n,
mus_model
)
def nearest_movie(movie, nearest_n):
return get_nearest_(
movie, "movie", cinema_mapping, rev_cinema_mapping, nearest_n, cinema_model
)
def nearest_plays(play, nearest_n):
return get_nearest_(
play, "plays", plays_mapping, rev_plays_mapping, nearest_n, plays_model
)
def nearest_excursion(excursion, nearest_n):
return get_nearest_(
excursion,
"excursion",
excursion_mapping,
rev_excursion_mapping,
nearest_n,
excursion_model,
)
def nearest_concert(concert, nearest_n):
return get_nearest_(
concert,
"concert",
concert_mapping,
rev_concert_mapping,
nearest_n,
concert_model,
)
def get_nearest_event(event, nearest_n):
if event.type == "plays":
return nearest_plays(event, nearest_n)
if event.type == "concert":
return nearest_concert(event, nearest_n)
if event.type == "movie":
return nearest_movie(event, nearest_n)
if event.type == 'museum':
return nearest_mus(event, nearest_n)
if event.type == 'attraction':
return nearest_attraction(event, nearest_n)
def update_preferences_state(user, event, direction):
pref = UserPreferences.objects.get(user=user)
if direction == "left":
if event.type == "plays":
pref.unpreffered_plays.add(event)
if event.type == "movie":
pref.unpreffered_movies.add(event)
if event.type == "concert":
pref.unpreferred_concerts.add(event)
else:
if event.type == "plays":
pref.preffered_plays.add(event)
if event.type == "movie":
pref.preffered_movies.add(event)
if event.type == "concert":
pref.preferred_concerts.add(event)
pref.save()
def get_next_tinder(user, prev_event, prev_direction):
pref = UserPreferences.objects.get(user=user)
print(prev_event.type, len(pref.preferred_concerts.all()))
if prev_direction == "left":
if prev_event.type == "plays" and len(pref.unpreffered_plays.all()) <= 2:
candidates = nearest_plays(prev_event, 100)
# print(candidates, type(candidates), len(Event.objects.filter(type='plays')))
return candidates[-1]
if prev_event.type == "movie" and len(pref.unpreffered_movies.all()) <= 2:
candidates = nearest_movie(prev_event, 100)
return candidates[-1]
if prev_event.type == "concert" and len(pref.unpreferred_concerts.all()) <= 2:
candidates = nearest_concert(prev_event, 100)
return candidates[-1]
if prev_direction == "right":
if prev_event.type == "plays" and len(pref.preffered_plays.all()) < 2:
candidates = nearest_plays(prev_event, 2)
return candidates[1]
if prev_event.type == "movie" and len(pref.preffered_movies.all()) < 2:
candidates = nearest_movie(prev_event, 2)
return candidates[1]
if prev_event.type == "concert" and len(pref.preferred_concerts.all()) < 2:
candidates = nearest_concert(prev_event, 2)
return candidates[1]
if prev_event.type == "plays":
if not len(pref.preffered_movies.all()) and not len(
pref.unpreffered_movies.all()
):
return choice(Event.objects.filter(type="movie"))
if not len(pref.preferred_concerts.all()) and not len(
pref.unpreferred_concerts.all()
):
return choice(Event.objects.filter(type="concert"))
if prev_event.type == "movie":
if not len(pref.preffered_plays.all()) and not len(
pref.unpreffered_plays.all()
):
return choice(Event.objects.filter(type="plays"))
if not len(pref.preferred_concerts.all()) and not len(
pref.unpreferred_concerts.all()
):
return choice(Event.objects.filter(type="concert"))
if prev_event.type == "concert":
if not len(pref.preffered_plays.all()) and not len(
pref.unpreffered_plays.all()
):
return choice(Event.objects.filter(type="plays"))
if not len(pref.preffered_movies.all()) and not len(
pref.unpreffered_movies.all()
):
return choice(Event.objects.filter(type="movie"))
return None
def rank_candidates(candidates_list, negative_candidates_list):
flatten_c_list = []
ranks = {}
flatten_negatives = []
for negative in negative_candidates_list:
flatten_negatives.extend(negative)
for lst in candidates_list:
flatten_c_list.extend(lst)
for cand in lst:
ranks.update({cand: {"rank": 0, "lst": lst}})
cnt = Counter(flatten_c_list)
for candidate, how_many in cnt.most_common(len(flatten_c_list)):
ranks[candidate]["rank"] = how_many * (
len(ranks[candidate]["lst"]) - ranks[candidate]["lst"].index(candidate)
)
res = []
for cand in ranks.keys():
res.append((ranks[cand]["rank"], cand))
return list(
filter(
lambda x: x[1] not in flatten_negatives, sorted(res, key=lambda x: -x[0])
)
)
def get_personal_recommendation(prefer, unprefer):
candidates = []
negative_candidates = []
for rec in prefer:
candidates.append(list(map(lambda x: x.oid, get_nearest_event(rec, 10)[1:])))
for neg in unprefer:
negative_candidates.append(
list(map(lambda x: x.oid, get_nearest_event(neg, 10)[1:]))
)
ranked = rank_candidates(candidates, negative_candidates)
return list(map(lambda x: (x[0], Event.objects.get(oid=x[1])), ranked[0:5]))
def get_personal_plays_recommendation(user):
pref = UserPreferences.objects.get(user=user)
prefer = pref.preffered_plays.all()
unprefer = pref.unpreffered_plays.all()
return get_personal_recommendation(prefer, unprefer)
def get_personal_concerts_recommendation(user):
pref = UserPreferences.objects.get(user=user)
prefer = pref.preferred_concerts.all()
unprefer = pref.unpreferred_concerts.all()
return get_personal_recommendation(prefer, unprefer)
def get_personal_movies_recommendation(user):
pref = UserPreferences.objects.get(user=user)
prefer = pref.preffered_movies.all()
unprefer = pref.unpreffered_movies.all()
return get_personal_recommendation(prefer, unprefer)
def dist_func(event1: Event, event2: Event):
cords1 = [event1.lat, event1.lon]
cords2 = [event2.lat, event2.lon]
try:
dist = GD(cords1, cords2).km
return dist
except:
return 1000000
return (event1.lon - event2.lon) ** 2 + (event1.lat - event2.lat) ** 2
def generate_nearest():
NearestEvent.objects.all().delete()
all_events = list(Event.objects.all())
for i, event in enumerate(Event.objects.all()):
event_all_events = list(
sorted(all_events.copy(), key=lambda x: dist_func(event, x))
)
nearest = NearestEvent.objects.create(event=event)
nearest.nearest.set(event_all_events[0:100])
nearest.save()
if i % 10 == 0:
print(i)
def generate_nearest_rest():
NearestEventToRestaurant.objects.all().delete()
all_events = list(Event.objects.all())
for i, rest in enumerate(Restaurant.objects.all()):
sorted_events = list(
sorted(
all_events.copy(),
key=lambda event: dist_func(rest, event)
)
)
nearest = NearestEventToRestaurant.objects.create(restaurant=rest)
nearest.events.set(sorted_events[0:100])
if i % 10 == 0:
print(i)
def generate_hotel_nearest():
NearestHotel.objects.all().delete()
all_events = list(Event.objects.all())
hotels = list(Hotel.objects.all())
for i, hotel in enumerate(hotels):
event_all_events = list(
sorted(all_events.copy(), key=lambda x: dist_func(hotel, x))
)
nearest = NearestHotel.objects.create(hotel=hotel)
nearest.nearest_events.set(event_all_events[0:100])
if i % 10 == 0:
print(i)
def generate_nearest_restaurants():
rests = list(Restaurant.objects.all())
for i, event in enumerate(Event.objects.all()):
sorted_rests = list(sorted(rests.copy(), key=lambda x: dist_func(x, event)))
nr = NearestRestaurantToEvent.objects.create(event=event)
nr.restaurants.set(sorted_rests[0:20])
nr.save()
if i % 10 == 0:
print(i)
for i, hotel in enumerate(Hotel.objects.all()):
sorted_rests = list(sorted(rests.copy(), key=lambda x: dist_func(x, hotel)))
nr = NearestRestaurantToHotel.objects.create(hotel=hotel)
nr.restaurants.set(sorted_rests[0:20])
nr.save()
if i % 10 == 0:
print(i)
def match_points():
regions = list(City.objects.all())
for i, point in enumerate(Event.objects.all()):
s_regions = list(sorted(regions.copy(), key=lambda x: dist_func(point, x)))
point.city = s_regions[0]
point.save()
if i % 10 == 0:
print(i)
for i, point in enumerate(Hotel.objects.all()):
s_regions = list(sorted(regions.copy(), key=lambda x: dist_func(point, x)))
point.city = s_regions[0]
point.save()
if i % 10 == 0:
print(i)
def match_restaurants():
regions = list(City.objects.all())
for i, point in enumerate(Restaurant.objects.all()):
s_regions = list(sorted(regions.copy(), key=lambda x: dist_func(point, x)))
point.city = s_regions[0]
point.save()
if i % 10 == 0:
print(i)
def calculate_mean_metric(
favorite_events: Iterable[Event],
target_event: Event,
model: AnnoyIndex,
rev_mapping,
):
if not len(favorite_events):
return 100000
dists = []
try:
target_event_idx = rev_mapping[target_event.oid]
except:
return 10
for fav in favorite_events:
dists.append(model.get_distance(rev_mapping[fav.oid], target_event_idx))
return sum(dists) / len(dists)
def calculate_favorite_metric(event: Event, user: User):
pref = UserPreferences.objects.get(user=user)
if event.type == "plays":
preferred = pref.preffered_plays.all()
return calculate_mean_metric(preferred, event, plays_model, rev_plays_mapping)
if event.type == "concert":
preferred = pref.preferred_concerts.all()
return calculate_mean_metric(
preferred, event, concert_model, rev_concert_mapping
)
if event.type == "movie":
preferred = pref.preffered_movies.all()
return calculate_mean_metric(preferred, event, cinema_model, rev_cinema_mapping)
if event.type == 'attraction':
preferred = pref.prefferred_attractions.all()
return calculate_mean_metric(preferred, event, attracion_model, rev_attraction_mapping)
if event.type == 'museum':
preferred = pref.prefferred_museums.all()
return calculate_mean_metric(preferred, event, mus_model, rev_mus_mapping)
return 10
def get_exponential_koef(time: timedelta):
time = time.seconds
if time < 60 * 10:
return 2
if time < 60 * 20:
return 5
if time < 60 * 30:
return 10
if time < 60 * 40:
return 20
return int(1e10)
def get_category_similarity_coef(event, user):
up, _ = UserPreferences.objects.get_or_create(user=user)
cat = up.preferred_categories
if event.type in cat:
return 0.7
else:
return 1.2
def get_nearest_favorite(
events: Iterable[Event],
user: User,
base_event: Event,
exclude_events: Iterable[Event] = [],
velocity=3.0,
top_k=1
):
sorted_events = list(
sorted(
filter(lambda event: event not in exclude_events, events),
key=lambda event:
calculate_favorite_metric(event, user) *
get_exponential_koef(
time_func(
dist_func(
event, base_event
),
velocity
)
) *
get_category_similarity_coef(event, user)
)
)
if top_k == 1:
return sorted_events[0]
return sorted_events[0:top_k]
def filter_hotel(region: Region, user: User, stars: Iterable[int]):
hotels = Hotel.objects.filter(city=region)
return choice(hotels)
def time_func(km_distance: float, velocity: float):
return timedelta(minutes=(km_distance) / (velocity / 60))
def generate_route(point1: BasePoint, point2: BasePoint, velocity):
distance = dist_func(point1, point2)
time = time_func(distance, velocity)
return {
"type": "transition",
"distance": distance,
"time": time.seconds,
}
def generate_point(point: BasePoint):
event_data = ObjectRouteSerializer(point).data
return {
"type": "point",
"point": event_data,
"point_type": "point",
"time": timedelta(minutes=90+choice(range(-10, 90, 10))).seconds,
"distance": 0
}
def generate_restaurant(point: BasePoint):
rest_data = ObjectRouteSerializer(point).data
return {
"type": "point",
"point": rest_data,
"point_type": "restaurant",
"time": timedelta(minutes=90+choice(range(-10, 90, 10))).seconds
}
def generate_multiple_tours(user: User, city: City, start_date: datetime.date, end_date: datetime.date):
hotels = sample(list(Hotel.objects.filter(city=city)), 5)
pool = Pool(5)
return pool.map(generate_tour, [(user, start_date, end_date, hotel) for hotel in hotels])
def generate_tour(
user: User,
city: City,
start_date: datetime.date,
end_date: datetime.date,
avg_velocity=3.0,
stars=[],
hotel_type=['hotel', 'hostel', 'apartment'],
where_eat=['restaurant', 'bar', 'cafe'],
what_to_see=['attractions', 'museum', 'movie', 'concert', 'artwork', 'plays', 'shop', 'gallery', 'theme_park', 'viewpoint', 'zoo']
):
UserPreferences.objects.get_or_create(user=user)
hotels_candidates = Hotel.objects.filter(city=city)
if len(hotels_candidates.filter(stars__in=stars)):
hotels_candidates = hotels_candidates.filter(stars__in=stars)
try:
hotel = choice(list(hotels_candidates))
except:
hotel = city
current_date = start_date
paths, points, disallowed_rest = [], [], []
while current_date < end_date:
local_points, local_paths, local_disallowed_rest = generate_path(
user,
points,
hotel,
disallowed_rest,
avg_velocity,
where_eat=where_eat,
what_to_see=what_to_see
)
points.extend(local_points)
paths.append(
{
'date': current_date,
'paths': local_paths
}
)
disallowed_rest = local_disallowed_rest
current_date += timedelta(days=1)
return paths, points
def generate_hotel(hotel: Hotel):
hotel_data = ObjectRouteSerializer(hotel).data
return {
"type": "point",
"point": hotel_data,
"point_type": "hotel",
"time": 0,
"distance": 0
}
def nearest_distance_points(point: BasePoint, user: User, velocity: float=3.0):
nearest = []
print(isinstance(point, Event), point)
if isinstance(point, Event):
nearest = NearestEvent.objects.get(event=point).nearest.all()
if isinstance(point, Hotel):
nearest = NearestHotel.objects.get(hotel=point).nearest_events.all()
if isinstance(point, Restaurant):
nearest = NearestEventToRestaurant.objects.get(restaurant=point).events.all()
top_nearest = get_nearest_favorite(nearest, user, point, [], velocity, top_k=10)
return top_nearest
def generate_path(
user: User,
disallowed_points: Iterable[BasePoint],
hotel: Hotel,
disallowed_rests: Iterable[Restaurant],
avg_velocity: float,
where_eat=['restaurant', 'bar', 'cafe'],
what_to_see=['attractions', 'museum', 'movie', 'concert', 'artwork', 'plays', 'shop', 'gallery', 'theme_park', 'viewpoint', 'zoo']
):
allowed_types = [
'museum',
'attraction',
'artwork',
'shop',
'gallery',
'theme_park',
'zoo',
'other',
'viewpoint'
]
if len(set(allowed_types) & set(what_to_see)) == 0:
allowed_types = what_to_see
else:
allowed_types = list(set(allowed_types) & set(what_to_see))
print(allowed_types, hotel)
if isinstance(hotel, City):
start_points_candidate = Restaurant.objects.filter(city=hotel).filter(~Q(oid__in=disallowed_rests))
else:
start_points_candidate = NearestRestaurantToHotel.objects.filter(hotel=hotel).first().restaurants.filter(~Q(oid__in=disallowed_rests))
if len(start_points_candidate.filter(type__in=where_eat)):
start_points_candidate = start_points_candidate.filter(type__in=where_eat)
start_point = start_points_candidate[0]
disallowed_rests.append(start_point.oid)
candidates = NearestEventToRestaurant.objects.get(restaurant=start_point).events.all().filter(type__in=allowed_types)
points = [start_point]
if isinstance(hotel, Hotel):
path = [
generate_hotel(hotel),
generate_route(start_point, hotel, avg_velocity),
generate_restaurant(start_point)
]
else:
path = [
generate_restaurant(start_point)
]
start_time = datetime.combine(datetime.now(), time(hour=10))
how_many_eat = 1
while start_time.hour < 22 and start_time.day == datetime.now().day:
if (start_time.hour > 14 and how_many_eat == 1) or (start_time.hour > 20 and how_many_eat == 2):
print(points, start_time)
try:
point_candidates = NearestRestaurantToEvent.objects.filter(event=points[-1]).first().restaurants.filter(~Q(oid__in=disallowed_rests))
if len(point_candidates.filter(type__in=where_eat)):
point_candidates = point_candidates.filter(type__in=where_eat)
point = point_candidates[0]
disallowed_rests.append(point.oid)
points.append(point)
candidates = NearestEventToRestaurant.objects.get(restaurant=point).events.all().filter(type__in=allowed_types)
if len(candidates) < 2:
candidates = NearestEventToRestaurant.objects.get(restaurant=point).events.all()
path.append(generate_restaurant(points[-1]))
start_time += timedelta(seconds=path[-1]['time'])
how_many_eat += 1
continue
except:
return points, path, disallowed_rests
if start_time.hour > 17:
allowed_types = [
'play',
'concert',
'movie',
'shop',
'gallery',
'theme_park',
'viewpoint'
]
if len(set(allowed_types) & set(what_to_see)) == 0:
allowed_types = what_to_see
else:
allowed_types = list(set(allowed_types) & set(what_to_see))
if candidates is None:
candidates = NearestEvent.objects.get(event=points[-1]).nearest.filter(type__in=allowed_types)
if len(candidates) < 2:
candidates = NearestEvent.objects.get(event=points[-1]).nearest.all()
try:
points.append(get_nearest_favorite(candidates, user, points[-1], points + disallowed_points))
except:
points.append(get_nearest_favorite(candidates, user, points[-1], points))
transition_route = generate_route(points[-1], points[-2], avg_velocity)
start_time += timedelta(seconds=transition_route["time"])
point_route = generate_point(points[-1])
start_time += timedelta(seconds=point_route["time"])
path.extend([transition_route, point_route])
candidates = None
return points, path, disallowed_rests
def calculate_distance(
sample1: Event, samples: Iterable[Event], model: AnnoyIndex, rev_mapping
):
metrics = []
for sample in samples:
metrics.append(
model.get_distance(rev_mapping[sample1.oid], rev_mapping[sample.oid])
)
return sum(metrics) / len(metrics)
def get_onboarding_attractions():
sample_attractions = sample(list(Event.objects.filter(type="attraction")), 200)
first_attraction = choice(sample_attractions)
attractions = [first_attraction]
while len(attractions) < 10:
mx_dist = 0
mx_attraction = None
for att in sample_attractions:
if att in attractions: continue
local_dist = calculate_distance(
att,
attractions,
attracion_model,
rev_attraction_mapping
)
if local_dist > mx_dist:
mx_dist = local_dist
mx_attraction = att
attractions.append(mx_attraction)
return attractions
def get_onboarding_hotels(stars=Iterable[int]):
return sample(list(Hotel.objects.filter(stars__in=stars)), 10)
def generate_points_path(user: User, points: Iterable[Event], velocity=3.0):
if len(points) < 7:
candidates = NearestEvent.objects.get(event=points[0]).nearest.all()
points.extend(list(get_nearest_favorite(candidates, user, points[0], [], velocity, 7-len(points))))
dist_matrix = [[0 for j in range(len(points))] for i in range(len(points))]
for i in range(len(dist_matrix)):
for j in range(len(dist_matrix)):
dist_matrix[i][j] = time_func(dist_func(points[i], points[j]), velocity).seconds
for i in range(len(dist_matrix)):
dist_matrix[i][0] = 0
dist_matrix = np.array(dist_matrix)
dist_matrix[:, 0] = 0
perm, dist = solve_tsp_dynamic_programming(dist_matrix)
perm_pts = [points[i] for i in perm]
res = [generate_point(perm_pts[0])]
visited_points = [perm_pts[0]]
for pt in perm_pts[1:]:
res.extend([
generate_route(
visited_points[-1],
pt,
velocity
),
generate_point(pt)
])
visited_points.append(pt)
return res
def flat_list(lst):
res = []
for i in lst:
res.extend(i)
return res
def range_candidates(candidates, user, favorite_events):
model_mappings = {
'attraction': [attracion_model, rev_attraction_mapping],
'museum': [mus_model, rev_mus_mapping],
'movie': [cinema_model, rev_cinema_mapping],
'concert': [concert_model, rev_concert_mapping],
'plays': [plays_model, rev_plays_mapping]
}
if candidates[0].type in ['attraction', 'museum', 'movie', 'concert', 'plays']:
candidates = sorted(
candidates,
key=lambda cand: calculate_mean_metric(
favorite_events,
cand,
*model_mappings[cand.type]
)
)
return candidates[0:10]
return sample(candidates, 10)
def get_personal_recomendations(user):
up, _ = UserPreferences.objects.get_or_create(user=user)
candidates_generate_strategy = {
'plays': [lambda pref: flat_list(
list(
map(
lambda cand: nearest_plays(
cand, 30
),
pref.preffered_plays.all()
)
),
), lambda pref: pref.preffered_plays.all()],
'movie': [lambda pref: flat_list(
list(
map(
lambda cand: nearest_movie(
cand, 30
),
pref.preffered_movies.all()
)
),
), lambda pref: pref.preffered_movies.all()],
'concert': [lambda pref: flat_list(
list(
map(
lambda cand: nearest_concert(
cand, 30
),
pref.preferred_concerts.all()
)
),
), lambda pref: pref.preferred_concerts.all()],
'attractions': [lambda pref: flat_list(
list(
map(
lambda cand: nearest_attraction(
cand, 30
),
pref.prefferred_attractions.all()
)
),
), lambda pref: pref.prefferred_attractions.all()],
'museum': [lambda pref: flat_list(
list(
map(
lambda cand: nearest_mus(
cand, 30
),
pref.prefferred_museums.all()
)
),
), lambda pref: pref.prefferred_museums.all()],
'shop': [lambda pref: sample(list(Event.objects.filter(type='shop')), 10), lambda x: []],
'gallery': [lambda pref: sample(list(Event.objects.filter(type='gallery')), 10), lambda x: []],
'theme_park': [lambda pref: sample(list(Event.objects.filter(type='theme_park')), 10), lambda x: []],
'viewpoint': [lambda pref: sample(list(Event.objects.filter(type='viewpoint')), 10), lambda x: []],
'zoo': [lambda pref: sample(list(Event.objects.filter(type='zoo')), 10), lambda x: []],
}
res = []
for category_candidate in up.preferred_categories:
candidates = candidates_generate_strategy[category_candidate][0](up)
ranged = range_candidates(
candidates,
user,
candidates_generate_strategy[category_candidate][1](up)
)
res.append(
{
'category': category_candidate,
'events': list(
map(
lambda x: ObjectRouteSerializer(x).data,
ranged
)
)
}
)
return res