Skip to content

Commit 3d9fe18

Browse files
muratcakirmuratcakir
committed
Added column mapping feature to data exchange framework
1 parent 033fb55 commit 3d9fe18

10 files changed

Lines changed: 180 additions & 36 deletions

File tree

‎src/Libraries/SmartStore.Core/ComponentModel/TypeConversion/DateTimeConverter.cs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ public override bool CanConvertFrom(Type type)
2626

2727
public override bool CanConvertTo(Type type)
2828
{
29-
return type == typeof(string)
29+
return type == typeof(string)
30+
|| type == typeof(long)
31+
|| type == typeof(double)
3032
|| type == typeof(DateTimeOffset)
3133
|| type == typeof(TimeSpan)
3234
|| base.CanConvertTo(type);
@@ -57,7 +59,7 @@ public override object ConvertFrom(CultureInfo culture, object value)
5759
}
5860

5961
double dbl;
60-
if (double.TryParse(str, NumberStyles.None, culture, out dbl))
62+
if (double.TryParse(str, NumberStyles.AllowDecimalPoint, culture, out dbl))
6163
{
6264
return DateTime.FromOADate(dbl);
6365
}
@@ -90,6 +92,16 @@ public override object ConvertTo(CultureInfo culture, string format, object valu
9092
return new TimeSpan(time.Ticks);
9193
}
9294

95+
if (to == typeof(double))
96+
{
97+
return time.ToOADate();
98+
}
99+
100+
if (to == typeof(long))
101+
{
102+
return time.ToUnixTime();
103+
}
104+
93105
return base.ConvertTo(culture, format, value, to);
94106
}
95107
}

‎src/Libraries/SmartStore.Services/DataExchange/ColumnMap.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace SmartStore.Services.DataExchange
8+
{
9+
public class ColumnMap
10+
{
11+
private readonly Dictionary<string, string> _map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
12+
13+
public IReadOnlyDictionary<string, string> Mappings
14+
{
15+
get { return _map; }
16+
}
17+
18+
public void AddMapping(string sourceName, string mappedName)
19+
{
20+
AddMapping(sourceName, null, mappedName);
21+
}
22+
23+
public void AddMapping(string sourceName, string index, string mappedName)
24+
{
25+
Guard.ArgumentNotEmpty(() => sourceName);
26+
Guard.ArgumentNotEmpty(() => mappedName);
27+
28+
if (index.HasValue())
29+
{
30+
sourceName += String.Concat("[", index, "]");
31+
}
32+
33+
_map[CreateName(sourceName, index)] = mappedName;
34+
}
35+
36+
/// <summary>
37+
/// Gets a mapped column name
38+
/// </summary>
39+
/// <param name="sourceName">The name of the column to get a mapped name for.</param>
40+
/// <returns>The mapped column name OR - if the name is unmapped - the passed <paramref name="sourceName"/></returns>
41+
public string GetMappedName(string sourceName)
42+
{
43+
string result;
44+
if (_map.TryGetValue(sourceName, out result))
45+
{
46+
return result;
47+
}
48+
49+
return sourceName;
50+
}
51+
52+
/// <summary>
53+
/// Gets a mapped column name
54+
/// </summary>
55+
/// <param name="sourceName">The name of the column to get a mapped name for.</param>
56+
/// <param name="index">The column index, e.g. a language code (de, en etc.)</param>
57+
/// <returns>The mapped column name OR - if the name is unmapped - the passed <paramref name="sourceName"/>[<paramref name="index"/>]</returns>
58+
public string GetMappedName(string sourceName, string index)
59+
{
60+
sourceName = CreateName(sourceName, index);
61+
62+
string result;
63+
if (_map.TryGetValue(sourceName, out result))
64+
{
65+
return result;
66+
}
67+
68+
return sourceName;
69+
}
70+
71+
internal static string CreateName(string name, string index)
72+
{
73+
if (index.HasValue())
74+
{
75+
name += String.Concat("[", index, "]");
76+
}
77+
78+
return name;
79+
}
80+
}
81+
}

‎src/Libraries/SmartStore.Services/DataExchange/Csv/CsvWriter.cs‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -149,27 +149,33 @@ public virtual void NextRow()
149149
_currentRow.Clear();
150150
}
151151

152-
private void WriteRow(string[] row)
152+
public string CurrentRawValue()
153+
{
154+
var row = string.Join(Configuration.Delimiter.ToString(), _currentRow);
155+
return row;
156+
}
157+
158+
private void WriteRow(string[] fields)
153159
{
154160
CheckDisposed();
155161

156-
if (row.Length == 0)
162+
if (fields.Length == 0)
157163
{
158164
throw new SmartException("Cannot write an empty row to the CSV file.");
159165
}
160166

161167
if (!_fieldCount.HasValue)
162168
{
163-
_fieldCount = row.Length;
169+
_fieldCount = fields.Length;
164170
}
165171

166-
if (_fieldCount.Value != row.Length)
172+
if (_fieldCount.Value != fields.Length)
167173
{
168174
throw new SmartException("The field count of the current row does not match the previous row's field count.");
169175
}
170176

171-
var rowString = string.Join(Configuration.Delimiter.ToString(), row);
172-
_writer.WriteLine(rowString);
177+
var row = string.Join(Configuration.Delimiter.ToString(), fields);
178+
_writer.WriteLine(row);
173179
}
174180

175181
protected override void OnDispose(bool disposing)

‎src/Libraries/SmartStore.Services/DataExchange/Import/DataTable/LightweightDataTable.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ public override IEnumerable<string> GetDynamicMemberNames()
267267
return _table.Columns.Select(x => x.Name);
268268
}
269269

270+
270271
public override bool TryGetMember(GetMemberBinder binder, out object result)
271272
{
272273
result = null;

‎src/Libraries/SmartStore.Services/DataExchange/Import/ImportDataSegmenter.cs‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ public class ImportDataSegmenter<T> where T : BaseEntity
1616
private ImportRow<T>[] _currentBatch;
1717
private IPageable _pageable;
1818
private bool _bof;
19+
private CultureInfo _culture;
20+
private ColumnMap _columnMap;
1921

2022
public ImportDataSegmenter(IDataTable table)
2123
{
@@ -25,13 +27,32 @@ public ImportDataSegmenter(IDataTable table)
2527

2628
_bof = true;
2729
_pageable = new PagedList(0, BATCHSIZE, table.Rows.Count);
28-
Culture = CultureInfo.InvariantCulture;
29-
}
30+
_culture = CultureInfo.InvariantCulture;
31+
_columnMap = new ColumnMap();
32+
}
3033

3134
public CultureInfo Culture
3235
{
33-
get;
34-
set;
36+
get
37+
{
38+
return _culture;
39+
}
40+
set
41+
{
42+
_culture = value ?? CultureInfo.InvariantCulture;
43+
}
44+
}
45+
46+
public ColumnMap ColumnMap
47+
{
48+
get
49+
{
50+
return _columnMap;
51+
}
52+
set
53+
{
54+
_columnMap = value ?? new ColumnMap();
55+
}
3556
}
3657

3758
public int TotalRows
@@ -64,6 +85,16 @@ public int BatchSize
6485
get { return BATCHSIZE; }
6586
}
6687

88+
public bool HasDataColumn(string name)
89+
{
90+
return _table.HasColumn(_columnMap.GetMappedName(name));
91+
}
92+
93+
public bool HasDataColumn(string name, string index)
94+
{
95+
return _table.HasColumn(_columnMap.GetMappedName(name, index));
96+
}
97+
6798
public void Reset()
6899
{
69100
if (_pageable.PageIndex != 0 && _currentBatch != null)
@@ -103,11 +134,11 @@ public ImportRow<T>[] CurrentBatch
103134
{
104135
if (_currentBatch == null)
105136
{
106-
_currentBatch = new ImportRow<T>[BATCHSIZE];
107-
108137
int start = _pageable.FirstItemIndex - 1;
109138
int end = _pageable.LastItemIndex - 1;
110139

140+
_currentBatch = new ImportRow<T>[(end - start) + 1];
141+
111142
// Determine values per row
112143
int i = 0;
113144
for (int r = start; r <= end; r++)

‎src/Libraries/SmartStore.Services/DataExchange/Import/ImportManager.cs‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@ public virtual ImportResult ImportProducts(
157157
try {
158158
var segmenter = new ImportDataSegmenter<Product>(table);
159159

160+
segmenter.Culture = CultureInfo.CurrentUICulture;
161+
160162
result.TotalRecords = segmenter.TotalRows;
161163

162164
while (segmenter.ReadNextBatch() && !cancellationToken.IsCancellationRequested)
@@ -335,19 +337,21 @@ private int ProcessProducts(ImportRow<Product>[] batch, ImportResult result)
335337
if (product == null)
336338
{
337339
// a Name is required with new products.
338-
if (!row.HasDataColumn("Name"))
340+
if (!row.Segmenter.HasDataColumn("Name"))
339341
{
340342
result.AddError("The 'Name' field is required for new products. Skipping row.", row.GetRowInfo(), "Name");
341343
continue;
342344
}
343345
product = new Product();
344346
}
345347

346-
row.Initialize(product, row.GetDataValue<string>("Name"));
348+
string name = row.GetDataValue<string>("Name");
349+
350+
row.Initialize(product, name);
347351

348352
if (!row.IsNew)
349353
{
350-
if (!product.Name.Equals(row.GetDataValue<string>("Name"), StringComparison.OrdinalIgnoreCase))
354+
if (!product.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
351355
{
352356
// Perf: use this later for SeName updates.
353357
row.NameChanged = true;
@@ -487,7 +491,7 @@ private int ProcessSlugs(ImportRow<Product>[] batch, ImportResult result)
487491

488492
foreach (var row in batch)
489493
{
490-
if (row.IsNew || row.NameChanged || row.HasDataColumn("SeName"))
494+
if (row.IsNew || row.NameChanged || row.Segmenter.HasDataColumn("SeName"))
491495
{
492496
try
493497
{
@@ -556,9 +560,9 @@ private int ProcessLocalizations(ImportRow<Product>[] batch, ImportResult result
556560

557561
foreach (var lang in languages)
558562
{
559-
string localizedName = row.GetDataValue<string>("Name[" + lang.UniqueSeoCode + "]");
560-
string localizedShortDescription = row.GetDataValue<string>("ShortDescription[" + lang.UniqueSeoCode + "]");
561-
string localizedFullDescription = row.GetDataValue<string>("FullDescription[" + lang.UniqueSeoCode + "]");
563+
string localizedName = row.GetDataValue<string>("Name", lang.UniqueSeoCode);
564+
string localizedShortDescription = row.GetDataValue<string>("ShortDescription", lang.UniqueSeoCode);
565+
string localizedFullDescription = row.GetDataValue<string>("FullDescription", lang.UniqueSeoCode);
562566

563567
if (localizedName.HasValue())
564568
{

‎src/Libraries/SmartStore.Services/DataExchange/Import/ImportRow.cs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,9 @@ public IDataRow DataRow
6060
get { return _row; }
6161
}
6262

63-
public bool HasDataColumn(string name)
63+
public ImportDataSegmenter<T> Segmenter
6464
{
65-
return _row.Table.HasColumn(name);
65+
get { return _segmenter; }
6666
}
6767

6868
public T Entity
@@ -87,9 +87,14 @@ public int Position
8787
}
8888

8989
public TProp GetDataValue<TProp>(string columnName)
90+
{
91+
return GetDataValue<TProp>(columnName, null);
92+
}
93+
94+
public TProp GetDataValue<TProp>(string columnName, string index)
9095
{
9196
object value;
92-
if (_row.TryGetValue(columnName, out value))
97+
if (_row.TryGetValue(_segmenter.ColumnMap.GetMappedName(columnName, index), out value))
9398
{
9499
return value.Convert<TProp>(_segmenter.Culture);
95100
}
@@ -115,7 +120,7 @@ public bool SetProperty<TProp>(
115120
var fastProp = FastProperty.GetProperty(target.GetUnproxiedType(), propName, PropertyCachingStrategy.EagerCached);
116121

117122
object value;
118-
if (_row.TryGetValue(propName, out value))
123+
if (_row.TryGetValue(_segmenter.ColumnMap.GetMappedName(propName), out value))
119124
{
120125
// source contains field value. Set it.
121126
TProp converted;

‎src/Libraries/SmartStore.Services/SmartStore.Services.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@
170170
<Compile Include="Common\TempFileCleanupTask.cs" />
171171
<Compile Include="Common\UAParserUserAgent.cs" />
172172
<Compile Include="Customers\CustomerRegisteredEvent.cs" />
173+
<Compile Include="DataExchange\ColumnMap.cs" />
173174
<Compile Include="DataExchange\Csv\CsvConfiguration.cs" />
174175
<Compile Include="DataExchange\Csv\CsvDataReader.cs" />
175176
<Compile Include="DataExchange\Csv\CsvWriter.cs" />

‎src/Tests/SmartStore.Core.Tests/SmartStore.Core.Tests.csproj‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@
111111
<None Include="packages.config" />
112112
</ItemGroup>
113113
<ItemGroup>
114-
<Folder Include="ComponentModel\" />
115114
<Folder Include="Infrastructure\DependencyManagement\Services\" />
116115
</ItemGroup>
117116
<ItemGroup>

‎src/Tests/SmartStore.Services.Tests/DataExchange/DataReaderTests.cs‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,23 @@ private void CreateDataTableFromReader()
5353
{
5454
IDataTable dataTable;
5555

56-
//using (var csv = new CsvDataReader(new StreamReader("D:\\products-10000.csv")))
57-
//{
58-
// dataTable = LightweightDataTable.FromDataReader(csv, 100, 500);
59-
//}
60-
61-
using (var fileStream = new FileStream("D:\\products-10000.xlsx", FileMode.Open, FileAccess.Read))
56+
using (var csv = new CsvDataReader(new StreamReader("D:\\csvwriter-1-7-0001-elmarshopinfocsvfeed.csv")))
6257
{
63-
using (var excel = new ExcelDataReader(fileStream, true))
64-
{
65-
dataTable = LightweightDataTable.FromDataReader(excel);
66-
}
58+
dataTable = LightweightDataTable.FromDataReader(csv);
59+
//foreach (var r in dataTable.Rows)
60+
//{
61+
// ObjectDumper.ToConsole(r);
62+
//}
6763
}
6864

65+
//using (var fileStream = new FileStream("D:\\products-10000.xlsx", FileMode.Open, FileAccess.Read))
66+
//{
67+
// using (var excel = new ExcelDataReader(fileStream, true))
68+
// {
69+
// dataTable = LightweightDataTable.FromDataReader(excel);
70+
// }
71+
//}
72+
6973
Console.WriteLine("TotalRows: " + dataTable.Rows.Count);
7074

7175
dynamic lastRow = dataTable.Rows.Last();

0 commit comments

Comments
 (0)