Warning: mkdir(): No space left on device in /var/www/hottg/post.php on line 59

Warning: file_put_contents(aCache/aDaily/2025-07-22/post/dbbooks/--): Failed to open stream: No such file or directory in /var/www/hottg/post.php on line 72
πŸš— Как Π½Π°ΠΉΡ‚ΠΈ ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ Apache Spark ΠΈ GraphFrames @Π‘Π°Π·Ρ‹ Π΄Π°Π½Π½Ρ‹Ρ…. Книги ΠΏΠΎ ΠΏΡ€ΠΎΠ³Ρ€Π°ΠΌΠΌΠΈΡ€ΠΎΠ²Π°Π½ΠΈΡŽ
TG Telegram Group & Channel
Π‘Π°Π·Ρ‹ Π΄Π°Π½Π½Ρ‹Ρ…. Книги ΠΏΠΎ ΠΏΡ€ΠΎΠ³Ρ€Π°ΠΌΠΌΠΈΡ€ΠΎΠ²Π°Π½ΠΈΡŽ | United States America (US)
Create: Update:

πŸš— Как Π½Π°ΠΉΡ‚ΠΈ ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ Apache Spark ΠΈ GraphFrames

Π Π°Π·Π±ΠΈΡ€Π°Π΅ΠΌ кСйс Π½Π° Ρ€Π΅Π°Π»ΡŒΠ½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ… ΠΈΠ· OpenStreetMap β€” ΠΈΡ‰Π΅ΠΌ ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚

πŸ” Π§Ρ‚ΠΎ Π΄Π΅Π»Π°Π΅ΠΌ
1. Π—Π°Π³Ρ€ΡƒΠΆΠ°Π΅ΠΌ Π³Ρ€Π°Ρ„ Π΄ΠΎΡ€ΠΎΠ³ Π³ΠΎΡ€ΠΎΠ΄Π° с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ OSMnx
2. БохраняСм Π²Π΅Ρ€ΡˆΠΈΠ½Ρ‹ ΠΈ Ρ€Π΅Π±Ρ€Π° с ΠΊΠΎΠΎΡ€Π΄ΠΈΠ½Π°Ρ‚Π°ΠΌΠΈ, скоростями ΠΈ Π³Π΅ΠΎΠΌΠ΅Ρ‚Ρ€ΠΈΠ΅ΠΉ
3. Π—Π°Π³Ρ€ΡƒΠΆΠ°Π΅ΠΌ всё Π² Spark
4. Находим ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΏΡƒΡ‚ΡŒ с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ GraphFrames

πŸ“ 1. Π‘ΠΊΠ°Ρ‡ΠΈΠ²Π°Π΅ΠΌ ΠΊΠ°Ρ€Ρ‚Ρƒ ΠΈ строим Π³Ρ€Π°Ρ„ ΡƒΠ»ΠΈΡ†

import osmnx as ox

# Π—Π°Π³Ρ€ΡƒΠ·ΠΊΠ° Π΄Π°Π½Π½Ρ‹Ρ… ΠΎ Π΄ΠΎΡ€ΠΎΠ³Π°Ρ… ΠœΠΎΡΠΊΠ²Ρ‹
G = ox.graph.graph_from_place("Moscow", network_type="drive")

# ΠžΡ‚ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ Π΄ΠΎΡ€ΠΎΠ³ Π½Π° ΠΊΠ°Ρ€Ρ‚Π΅
moscow_gdf = ox.geocoder.geocode_to_gdf("Moscow")
fig, ax = ox.plot.plot_graph(G, show=False, close=False, bgcolor="#111111", edge_color="#ffcb00", edge_linewidth=0.3, node_size=0)
moscow_gdf.plot(ax=ax, fc="#444444", ec=None, lw=1, alpha=1, zorder=-1)

# Настройка Π³Ρ€Π°Π½ΠΈΡ† ΠΊΠ°Ρ€Ρ‚Ρ‹
margin = 0.02
west, south, east, north = moscow_gdf.union_all().bounds
margin_ns = (north - south) * margin
margin_ew = (east - west) * margin
ax.set_ylim((south - margin_ns, north + margin_ns))
ax.set_xlim((west - margin_ew, east + margin_ew))
plt.show()


πŸ“ 2. БохраняСм гСомСтричСскоС описаниС Π³ΠΎΡ€ΠΎΠ΄Π° Π² Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π΅ GeoJSON ΠΈ Π΄Π°Π½Π½Ρ‹Π΅ ΠΎ Π²Π΅Ρ€ΡˆΠΈΠ½Π°Ρ… ΠΈ Ρ€Ρ‘Π±Ρ€Π°Ρ… Π² Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π΅ CSV
with open('Moscow.geojson', 'w') as file:
file.write(moscow_gdf.to_json())

nodes = G.nodes(data=True)
with open('nodes.csv', 'a') as file:
file.write("id,lat,lonn")
for (node, data) in nodes:
file.write("%d,%f,%fn" % (node, data.get("y"), data.get("x")))

edges = G.edges(data=True)
def decode_maxspeed(maxspeed):
match maxspeed:
case str():
match maxspeed.lower():
case "ru:urban": return 60
case "ru:rural": return 90
case "ru:living_street": return 20
case "ru:motorway": return 110
case _: return int(maxspeed)
case list(): return min(list(map(decode_maxspeed, maxspeed)))
case _: return maxspeed

with open('edges.csv', 'a') as file:
file.write("src,dst,maxspeed,length,geometryn")
for (src, dst, data) in edges:
maxspeed = decode_maxspeed(data.get("maxspeed", 999))
length = float(data.get("length"))
geometry = shapely.wkt.dumps(data.get("geometry"))
file.write("%d,%d,%d,%f,%sn" % (src, dst, maxspeed, length, geometry))


3. Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌ Π±ΠΈΠ±Π»ΠΈΠΎΡ‚Π΅ΠΊΡƒ GraphFrames для ΠΎΠ±Ρ€Π°Π±ΠΎΡ‚ΠΊΠΈ Π³Ρ€Π°Ρ„ΠΎΠ² Π½Π° Apache Spark

from pyspark.sql import SparkSession

spark = SparkSession.builder
.config("spark.jars.packages", "graphframes:graphframes:0.8.4-spark3.5-s_2.12")
.master("local[*]")
.appName("GraphFrames")
.getOrCreate()

nodes = spark.read.options(header=True).csv("nodes.csv")
edges = spark.read.options(header=True).csv("edges.csv")

# ВычислСниС Π²Ρ€Π΅ΠΌΠ΅Π½ΠΈ прохоТдСния Ρ€Ρ‘Π±Π΅Ρ€
edgesT = edges.withColumn("time", edges["length"] / edges["maxspeed"])

# ΠŸΠΎΡΡ‚Ρ€ΠΎΠ΅Π½ΠΈΠ΅ Π³Ρ€Π°Ρ„Π°
from graphframes import *

g = GraphFrame(nodes, edgesT)


🧭 4. Π˜Ρ‰Π΅ΠΌ ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΏΡƒΡ‚ΡŒ ΠΏΠΎ Π²Ρ€Π΅ΠΌΠ΅Π½ΠΈ
Π½Π°ΠΏΡ€ΠΈΠΌΠ΅Ρ€, ΠΎΡ‚ Измайлово Π΄ΠΎ Π–Πš Π—ΠΈΠ»Π°Ρ€Ρ‚
src = "257601812"
dst = "5840593081"

paths = g.shortestPaths(landmarks=[dst])
paths.filter(F.col("id") == src).show(truncate=False)


πŸ’‘ Π Π΅Π·ΡƒΠ»ΡŒΡ‚Π°Ρ‚: 40 шагов ΠΎΡ‚ Ρ‚ΠΎΡ‡ΠΊΠΈ A Π΄ΠΎ Ρ‚ΠΎΡ‡ΠΊΠΈ B.

Π’Π°ΠΊΠΎΠΉ ΠΏΠΎΠ΄Ρ…ΠΎΠ΄ Π»Π΅Π³ΠΊΠΎ ΠΌΠ°ΡΡˆΡ‚Π°Π±ΠΈΡ€ΡƒΠ΅Ρ‚ΡΡ Π½Π° ΠΌΠΈΠ»Π»ΠΈΠΎΠ½Ρ‹ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ΠΎΠ². Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠΉΡ‚Π΅ Spark ΠΈ GraphFrames для построСния логистичСских ΠΌΠΎΠ΄Π΅Π»Π΅ΠΉ, ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ΠΈΠ·Π°Ρ†ΠΈΠΈ ΠΈ городского планирования.

πŸš€ Π₯ΠΎΡ‚ΠΈΡ‚Π΅ ΠΏΡ€ΠΎΠΊΠ°Ρ‡Π°Ρ‚ΡŒΡΡ Π² Ρ€Π°Π±ΠΎΡ‚Π΅ с Big Data? Π˜Π·ΡƒΡ‡Π°ΠΉΡ‚Π΅ Spark! Π—Π°ΠΏΠΈΡΡ‹Π²Π°ΠΉΡ‚Π΅ΡΡŒ Π½Π° курс Spark Developer ΠΎΡ‚ OTUS β€” ΡƒΡ‡ΠΈΡ‚Π΅ΡΡŒ Π½Π° Ρ€Π΅Π°Π»ΡŒΠ½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ… ΠΈ ΠΏΡ€ΠΎΠ΄Π²ΠΈΠ½ΡƒΡ‚Ρ‹Ρ… кСйсах: https://vk.cc/cMT1Wg

Π Π΅ΠΊΠ»Π°ΠΌΠ°. ООО Β«ΠžΡ‚ΡƒΡ ΠΎΠ½Π»Π°ΠΉΠ½-ΠΎΠ±Ρ€Π°Π·ΠΎΠ²Π°Π½ΠΈΠ΅Β», ΠžΠ“Π Π 1177746618576

πŸš— Как Π½Π°ΠΉΡ‚ΠΈ ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ Apache Spark ΠΈ GraphFrames

Π Π°Π·Π±ΠΈΡ€Π°Π΅ΠΌ кСйс Π½Π° Ρ€Π΅Π°Π»ΡŒΠ½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ… ΠΈΠ· OpenStreetMap β€” ΠΈΡ‰Π΅ΠΌ ΠΎΠΏΡ‚ΠΈΠΌΠ°Π»ΡŒΠ½Ρ‹ΠΉ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚

πŸ” Π§Ρ‚ΠΎ Π΄Π΅Π»Π°Π΅ΠΌ
1. Π—Π°Π³Ρ€ΡƒΠΆΠ°Π΅ΠΌ Π³Ρ€Π°Ρ„ Π΄ΠΎΡ€ΠΎΠ³ Π³ΠΎΡ€ΠΎΠ΄Π° с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ OSMnx
2. БохраняСм Π²Π΅Ρ€ΡˆΠΈΠ½Ρ‹ ΠΈ Ρ€Π΅Π±Ρ€Π° с ΠΊΠΎΠΎΡ€Π΄ΠΈΠ½Π°Ρ‚Π°ΠΌΠΈ, скоростями ΠΈ Π³Π΅ΠΎΠΌΠ΅Ρ‚Ρ€ΠΈΠ΅ΠΉ
3. Π—Π°Π³Ρ€ΡƒΠΆΠ°Π΅ΠΌ всё Π² Spark
4. Находим ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΏΡƒΡ‚ΡŒ с ΠΏΠΎΠΌΠΎΡ‰ΡŒΡŽ GraphFrames

πŸ“ 1. Π‘ΠΊΠ°Ρ‡ΠΈΠ²Π°Π΅ΠΌ ΠΊΠ°Ρ€Ρ‚Ρƒ ΠΈ строим Π³Ρ€Π°Ρ„ ΡƒΠ»ΠΈΡ†

import osmnx as ox

# Π—Π°Π³Ρ€ΡƒΠ·ΠΊΠ° Π΄Π°Π½Π½Ρ‹Ρ… ΠΎ Π΄ΠΎΡ€ΠΎΠ³Π°Ρ… ΠœΠΎΡΠΊΠ²Ρ‹
G = ox.graph.graph_from_place("Moscow", network_type="drive")

# ΠžΡ‚ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ Π΄ΠΎΡ€ΠΎΠ³ Π½Π° ΠΊΠ°Ρ€Ρ‚Π΅
moscow_gdf = ox.geocoder.geocode_to_gdf("Moscow")
fig, ax = ox.plot.plot_graph(G, show=False, close=False, bgcolor="#111111", edge_color="#ffcb00", edge_linewidth=0.3, node_size=0)
moscow_gdf.plot(ax=ax, fc="#444444", ec=None, lw=1, alpha=1, zorder=-1)

# Настройка Π³Ρ€Π°Π½ΠΈΡ† ΠΊΠ°Ρ€Ρ‚Ρ‹
margin = 0.02
west, south, east, north = moscow_gdf.union_all().bounds
margin_ns = (north - south) * margin
margin_ew = (east - west) * margin
ax.set_ylim((south - margin_ns, north + margin_ns))
ax.set_xlim((west - margin_ew, east + margin_ew))
plt.show()


πŸ“ 2. БохраняСм гСомСтричСскоС описаниС Π³ΠΎΡ€ΠΎΠ΄Π° Π² Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π΅ GeoJSON ΠΈ Π΄Π°Π½Π½Ρ‹Π΅ ΠΎ Π²Π΅Ρ€ΡˆΠΈΠ½Π°Ρ… ΠΈ Ρ€Ρ‘Π±Ρ€Π°Ρ… Π² Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π΅ CSV
with open('Moscow.geojson', 'w') as file:
file.write(moscow_gdf.to_json())

nodes = G.nodes(data=True)
with open('nodes.csv', 'a') as file:
file.write("id,lat,lonn")
for (node, data) in nodes:
file.write("%d,%f,%fn" % (node, data.get("y"), data.get("x")))

edges = G.edges(data=True)
def decode_maxspeed(maxspeed):
match maxspeed:
case str():
match maxspeed.lower():
case "ru:urban": return 60
case "ru:rural": return 90
case "ru:living_street": return 20
case "ru:motorway": return 110
case _: return int(maxspeed)
case list(): return min(list(map(decode_maxspeed, maxspeed)))
case _: return maxspeed

with open('edges.csv', 'a') as file:
file.write("src,dst,maxspeed,length,geometryn")
for (src, dst, data) in edges:
maxspeed = decode_maxspeed(data.get("maxspeed", 999))
length = float(data.get("length"))
geometry = shapely.wkt.dumps(data.get("geometry"))
file.write("%d,%d,%d,%f,%sn" % (src, dst, maxspeed, length, geometry))


3. Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌ Π±ΠΈΠ±Π»ΠΈΠΎΡ‚Π΅ΠΊΡƒ GraphFrames для ΠΎΠ±Ρ€Π°Π±ΠΎΡ‚ΠΊΠΈ Π³Ρ€Π°Ρ„ΠΎΠ² Π½Π° Apache Spark

from pyspark.sql import SparkSession

spark = SparkSession.builder
.config("spark.jars.packages", "graphframes:graphframes:0.8.4-spark3.5-s_2.12")
.master("local[*]")
.appName("GraphFrames")
.getOrCreate()

nodes = spark.read.options(header=True).csv("nodes.csv")
edges = spark.read.options(header=True).csv("edges.csv")

# ВычислСниС Π²Ρ€Π΅ΠΌΠ΅Π½ΠΈ прохоТдСния Ρ€Ρ‘Π±Π΅Ρ€
edgesT = edges.withColumn("time", edges["length"] / edges["maxspeed"])

# ΠŸΠΎΡΡ‚Ρ€ΠΎΠ΅Π½ΠΈΠ΅ Π³Ρ€Π°Ρ„Π°
from graphframes import *

g = GraphFrame(nodes, edgesT)


🧭 4. Π˜Ρ‰Π΅ΠΌ ΠΊΡ€Π°Ρ‚Ρ‡Π°ΠΉΡˆΠΈΠΉ ΠΏΡƒΡ‚ΡŒ ΠΏΠΎ Π²Ρ€Π΅ΠΌΠ΅Π½ΠΈ
Π½Π°ΠΏΡ€ΠΈΠΌΠ΅Ρ€, ΠΎΡ‚ Измайлово Π΄ΠΎ Π–Πš Π—ΠΈΠ»Π°Ρ€Ρ‚
src = "257601812"
dst = "5840593081"

paths = g.shortestPaths(landmarks=[dst])
paths.filter(F.col("id") == src).show(truncate=False)


πŸ’‘ Π Π΅Π·ΡƒΠ»ΡŒΡ‚Π°Ρ‚: 40 шагов ΠΎΡ‚ Ρ‚ΠΎΡ‡ΠΊΠΈ A Π΄ΠΎ Ρ‚ΠΎΡ‡ΠΊΠΈ B.

Π’Π°ΠΊΠΎΠΉ ΠΏΠΎΠ΄Ρ…ΠΎΠ΄ Π»Π΅Π³ΠΊΠΎ ΠΌΠ°ΡΡˆΡ‚Π°Π±ΠΈΡ€ΡƒΠ΅Ρ‚ΡΡ Π½Π° ΠΌΠΈΠ»Π»ΠΈΠΎΠ½Ρ‹ ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ΠΎΠ². Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠΉΡ‚Π΅ Spark ΠΈ GraphFrames для построСния логистичСских ΠΌΠΎΠ΄Π΅Π»Π΅ΠΉ, ΠΌΠ°Ρ€ΡˆΡ€ΡƒΡ‚ΠΈΠ·Π°Ρ†ΠΈΠΈ ΠΈ городского планирования.

πŸš€ Π₯ΠΎΡ‚ΠΈΡ‚Π΅ ΠΏΡ€ΠΎΠΊΠ°Ρ‡Π°Ρ‚ΡŒΡΡ Π² Ρ€Π°Π±ΠΎΡ‚Π΅ с Big Data? Π˜Π·ΡƒΡ‡Π°ΠΉΡ‚Π΅ Spark! Π—Π°ΠΏΠΈΡΡ‹Π²Π°ΠΉΡ‚Π΅ΡΡŒ Π½Π° курс Spark Developer ΠΎΡ‚ OTUS β€” ΡƒΡ‡ΠΈΡ‚Π΅ΡΡŒ Π½Π° Ρ€Π΅Π°Π»ΡŒΠ½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ… ΠΈ ΠΏΡ€ΠΎΠ΄Π²ΠΈΠ½ΡƒΡ‚Ρ‹Ρ… кСйсах: https://vk.cc/cMT1Wg

Π Π΅ΠΊΠ»Π°ΠΌΠ°. ООО Β«ΠžΡ‚ΡƒΡ ΠΎΠ½Π»Π°ΠΉΠ½-ΠΎΠ±Ρ€Π°Π·ΠΎΠ²Π°Π½ΠΈΠ΅Β», ΠžΠ“Π Π 1177746618576
πŸ‘5❀4πŸ‘Ž2


>>Click here to continue<<

Π‘Π°Π·Ρ‹ Π΄Π°Π½Π½Ρ‹Ρ…. Книги ΠΏΠΎ ΠΏΡ€ΠΎΠ³Ρ€Π°ΠΌΠΌΠΈΡ€ΠΎΠ²Π°Π½ΠΈΡŽ




Share with your best friend
VIEW MORE

United States America Popular Telegram Group (US)


Warning: Undefined array key 3 in /var/www/hottg/function.php on line 115

Fatal error: Uncaught mysqli_sql_exception: Too many connections in /var/www/db.php:16 Stack trace: #0 /var/www/db.php(16): mysqli_connect() #1 /var/www/hottg/function.php(212): db() #2 /var/www/hottg/function.php(115): select() #3 /var/www/hottg/post.php(351): daCache() #4 /var/www/hottg/route.php(63): include_once('...') #5 {main} thrown in /var/www/db.php on line 16