ошибка цитирования при импорте форматированного столбца JSON географического объекта через CSV в rails + postgresql / postgis - PullRequest
Новая
       58

ошибка цитирования при импорте форматированного столбца JSON географического объекта через CSV в rails + postgresql / postgis

0 голосов
/ 10 апреля 2019

структура данных: 

create_table "regionpolygons", force: :cascade do |t|
  t.geometry "rawdata", limit: {:srid=>0, :type=>"geometry"}

Файл импорта CSV 

task :load_geometry_data  => :environment do

CSV.foreach("uploads/regions.tsv", :col_sep => "\t", headers: true) do |row|
   @polygon_json = row[0]
   @feature = RGeo::GeoJSON.decode(@polygon_json) 
   @polygon = @feature.geometry.as_text
   Regionpolygon.create(
      :rawdata => @polygon,
      [...]

не удается из-за CSV::MalformedCSVError: Illegal quoting in line 2., который я не вижу ( также попытался начать и завершить строку[0] с одинарной кавычкой ) и одной строкой в ​​файле импорта.строка 2 имеет (принудительный HR для разборчивости): 

{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[16.44,48.3072],[16.4416,48.3034],[16.4433,48.2912],[16.469,48.2776],[16.4796,48.2752],[16.4793,48.2897],[16.4854,48.2924],[16.5112,48.2865],[16.5138,48.2826],[16.508,48.2726],[16.5293,48.2609],[16.5467,48.2611],[16.5415,48.2422],[16.5526,48.2373],[16.5451,48.2236],[16.5472,48.2185],[16.5364,48.1999],[16.5399,48.1924],[16.5385,48.1797],[16.5481,48.1784],[16.5465,48.1703],[16.5524,48.1614],[16.5714,48.1617],[16.5783,48.1464],[16.576,48.1354],[16.5434,48.1419],[16.5315,48.1507],[16.5139,48.158],[16.5005,48.1555],[16.4855,48.1566],[16.4543,48.1377],[16.433,48.1361],[16.4334,48.1252],[16.4057,48.1198],[16.3877,48.1246],[16.3601,48.1281],[16.3283,48.1364],[16.3191,48.1218],[16.3125,48.1188],[16.2987,48.1273],[16.2707,48.1327],[16.2485,48.1286],[16.2357,48.1354],[16.2226,48.1341],[16.2103,48.1487],[16.1986,48.1544],[16.1934,48.1662],[16.1832,48.1711],[16.2008,48.187],[16.2019,48.2007],[16.1969,48.205],[16.1957,48.2422],[16.2073,48.262],[16.2365,48.2476],[16.2423,48.2379],[16.256,48.2406],[16.2698,48.2514],[16.2917,48.2627],[16.2879,48.269],[16.3054,48.2712],[16.3183,48.2787],[16.3474,48.2889],[16.3572,48.281],[16.3773,48.2894],[16.3729,48.2957],[16.3812,48.3063],[16.3754,48.3107],[16.3735,48.3134],[16.3859,48.3225],[16.3936,48.318],[16.419,48.3212],[16.4357,48.3155],[16.44,48.3072]]]},
"properties":{"name":"Vienna","id":"AT-9","CNTRYNAME":"Austria","TYPE":"State"},"id":"AT-9"}

пока из консоли, данные анализируются как предусмотрено: 

str4 = '{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[16.44,48.3072],[16.4416,48.3034],[16.4433,48.2912],[16.469,48.2776],[16.4796,48.2752],[16.4793,48.2897],[16.4854,48.2924],[16.5112,48.2865],[16.5138,48.2826],[16.508,48.2726],[16.5293,48.2609],[16.5467,48.2611],[16.5415,48.2422],[16.5526,48.2373],[16.5451,48.2236],[16.5472,48.2185],[16.5364,48.1999],[16.5399,48.1924],[16.5385,48.1797],[16.5481,48.1784],[16.5465,48.1703],[16.5524,48.1614],[16.5714,48.1617],[16.5783,48.1464],[16.576,48.1354],[16.5434,48.1419],[16.5315,48.1507],[16.5139,48.158],[16.5005,48.1555],[16.4855,48.1566],[16.4543,48.1377],[16.433,48.1361],[16.4334,48.1252],[16.4057,48.1198],[16.3877,48.1246],[16.3601,48.1281],[16.3283,48.1364],[16.3191,48.1218],[16.3125,48.1188],[16.2987,48.1273],[16.2707,48.1327],[16.2485,48.1286],[16.2357,48.1354],[16.2226,48.1341],[16.2103,48.1487],[16.1986,48.1544],[16.1934,48.1662],[16.1832,48.1711],[16.2008,48.187],[16.2019,48.2007],[16.1969,48.205],[16.1957,48.2422],[16.2073,48.262],[16.2365,48.2476],[16.2423,48.2379],[16.256,48.2406],[16.2698,48.2514],[16.2917,48.2627],[16.2879,48.269],[16.3054,48.2712],[16.3183,48.2787],[16.3474,48.2889],[16.3572,48.281],[16.3773,48.2894],[16.3729,48.2957],[16.3812,48.3063],[16.3754,48.3107],[16.3735,48.3134],[16.3859,48.3225],[16.3936,48.318],[16.419,48.3212],[16.4357,48.3155],[16.44,48.3072]]]},"properties":{"name":"Vienna","id":"AT-9","CNTRYNAME":"Austria","TYPE":"State"},"id":"AT-9"}'
=> "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[16.44,48.3072],[16.4416,48.3034],[16.4433,48.2912],[16.469,48.2776],[16.4796,48.2752],[16.4793,48.2897],[16.4854,48.2924],[16.5112,48.2865],[16.5138,48.2826],[16.508,48.2726],[16.5293,48.2609],[16.5467,48.2611],[16.5415,48.2422],[16.5526,48.2373],[16.5451,48.2236],[16.5472,48.2185],[16.5364,48.1999],[16.5399,48.1924],[16.5385,48.1797],[16.5481,48.1784],[16.5465,48.1703],[16.5524,48.1614],[16.5714,48.1617],[16.5783,48.1464],[16.576,48.1354],[16.5434,48.1419],[16.5315,48.1507],[16.5139,48.158],[16.5005,48.1555],[16.4855,48.1566],[16.4543,48.1377],[16.433,48.1361],[16.4334,48.1252],[16.4057,48.1198],[16.3877,48.1246],[16.3601,48.1281],[16.3283,48.1364],[16.3191,48.1218],[16.3125,48.1188],[16.2987,48.1273],[16.2707,48.1327],[16.2485,48.1286],[16.2357,48.1354],[16.2226,48.1341],[16.2103,48.1487],[16.1986,48.1544],[16.1934,48.1662],[16.1832,48.1711],[16.2008,48.187],[16.2019,48.2007],[16.1969,48.205],[16.1957,48.2422],[16.2073,48.262],[16.2365,48.2476],[16.2423,48.2379],[16.256,48.2406],[16.2698,48.2514],[16.2917,48.2627],[16.2879,48.269],[16.3054,48.2712],[16.3183,48.2787],[16.3474,48.2889],[16.3572,48.281],[16.3773,48.2894],[16.3729,48.2957],[16.3812,48.3063],[16.3754,48.3107],[16.3735,48.3134],[16.3859,48.3225],[16.3936,48.318],[16.419,48.3212],[16.4357,48.3155],[16.44,48.3072]]]},\"properties\":{\"name\":\"Vienna\",\"id\":\"AT-9\",\"CNTRYNAME\":\"Austria\",\"TYPE\":\"State\"},\"id\":\"AT-9\"}"
irb(main):051:0> feature = RGeo::GeoJSON.decode(str4)
=> #<RGeo::GeoJSON::Feature:0x2b12eb1c0920 id="AT-9" geom="POLYGON ((16.44 48.3072, 16.4416 48.3034, 16.4433 48.2912, 16.469 48.2776, 16.4796 48.2752, 16.4793 48.2897, 16.4854 48.2924, 16.5112 48.2865, 16.5138 48.2826, 16.508 48.2726, 16.5293 48.2609, 16.5467 48.2611, 16.5415 48.2422, 16.5526 48.2373, 16.5451 48.2236, 16.5472 48.2185, 16.5364 48.1999, 16.5399 48.1924, 16.5385 48.1797, 16.5481 48.1784, 16.5465 48.1703, 16.5524 48.1614, 16.5714 48.1617, 16.5783 48.1464, 16.576 48.1354, 16.5434 48.1419, 16.5315 48.1507, 16.5139 48.158, 16.5005 48.1555, 16.4855 48.1566, 16.4543 48.1377, 16.433 48.1361, 16.4334 48.1252, 16.4057 48.1198, 16.3877 48.1246, 16.3601 48.1281, 16.3283 48.1364, 16.3191 48.1218, 16.3125 48.1188, 16.2987 48.1273, 16.2707 48.1327, 16.2485 48.1286, 16.2357 48.1354, 16.2226 48.1341, 16.2103 48.1487, 16.1986 48.1544, 16.1934 48.1662, 16.1832 48.1711, 16.2008 48.187, 16.2019 48.2007, 16.1969 48.205, 16.1957 48.2422, 16.2073 48.262, 16.2365 48.2476, 16.2423 48.2379, 16.256 48.2406, 16.2698 48.2514, 16.2917 48.2627, 16.2879 48.269, 16.3054 48.2712, 16.3183 48.2787, 16.3474 48.2889, 16.3572 48.281, 16.3773 48.2894, 16.3729 48.2957, 16.3812 48.3063, 16.3754 48.3107, 16.3735 48.3134, 16.3859 48.3225, 16.3936 48.318, 16.419 48.3212, 16.4357 48.3155, 16.44 48.3072))">
irb(main):052:0> feature.geometry.as_text
=> "POLYGON ((16.44 48.3072, 16.4416 48.3034, 16.4433 48.2912, 16.469 48.2776, 16.4796 48.2752, 16.4793 48.2897, 16.4854 48.2924, 16.5112 48.2865, 16.5138 48.2826, 16.508 48.2726, 16.5293 48.2609, 16.5467 48.2611, 16.5415 48.2422, 16.5526 48.2373, 16.5451 48.2236, 16.5472 48.2185, 16.5364 48.1999, 16.5399 48.1924, 16.5385 48.1797, 16.5481 48.1784, 16.5465 48.1703, 16.5524 48.1614, 16.5714 48.1617, 16.5783 48.1464, 16.576 48.1354, 16.5434 48.1419, 16.5315 48.1507, 16.5139 48.158, 16.5005 48.1555, 16.4855 48.1566, 16.4543 48.1377, 16.433 48.1361, 16.4334 48.1252, 16.4057 48.1198, 16.3877 48.1246, 16.3601 48.1281, 16.3283 48.1364, 16.3191 48.1218, 16.3125 48.1188, 16.2987 48.1273, 16.2707 48.1327, 16.2485 48.1286, 16.2357 48.1354, 16.2226 48.1341, 16.2103 48.1487, 16.1986 48.1544, 16.1934 48.1662, 16.1832 48.1711, 16.2008 48.187, 16.2019 48.2007, 16.1969 48.205, 16.1957 48.2422, 16.2073 48.262, 16.2365 48.2476, 16.2423 48.2379, 16.256 48.2406, 16.2698 48.2514, 16.2917 48.2627, 16.2879 48.269, 16.3054 48.2712, 16.3183 48.2787, 16.3474 48.2889, 16.3572 48.281, 16.3773 48.2894, 16.3729 48.2957, 16.3812 48.3063, 16.3754 48.3107, 16.3735 48.3134, 16.3859 48.3225, 16.3936 48.318, 16.419 48.3212, 16.4357 48.3155, 16.44 48.3072))"
irb(main):053:0> feature['id']
=> "AT-9"
irb(main):054:0> feature['name']
=> "Vienna"

1 Ответ

0 голосов
/ 11 апреля 2019

Использование CSV на самом деле заставляет проходить бессмысленные обручи. Использовать обычный старый Ruby намного проще, учитывая расширение 

task :load_geo_data => :environment do
  tempdir = File.absolute_path('uploads/regions.tsv')
  File.open(tempdir).each do |line|
    feature = RGeo::GeoJSON.decode(line)
    Regionpolygon.create(
       rawdata: feature.geometry.as_text,
       [...]
     )
  end
end
...