C# serialize big collection via NewtonSoft.Json

System.OutOfMemoryException
  HResult=0x8007000E
  Message=Exception of type 'System.OutOfMemoryException' was thrown.
  Source=mscorlib
  StackTrace:
   at System.Text.StringBuilder.ToString()
   at System.IO.StringWriter.ToString()
   at Newtonsoft.Json.JsonConvert.SerializeObjectInternal(Object value, Type type, JsonSerializer jsonSerializer)
   at Newtonsoft.Json.JsonConvert.SerializeObject(Object value, Formatting formatting)
   at ConsoleApp3.Program.TestBigDataSerialize() in D:\C\ConsoleApp3\ConsoleApp3\Program.cs:line 63
   at ConsoleApp3.Program.Main(String[] args) in D:\C\ConsoleApp3\ConsoleApp3\Program.cs:line 29

 

 

The solution is extend stream

public static class BigDataSerializeStream
{
    public static void WriteJsonToStream<T>(this Stream stream,IEnumerable<T> items)
    {
        using(var streamWriter=new StreamWriter(stream))
        {
            using(var jsonWriter=new JsonTextWriter(streamWriter))
            {  
                var serializer = new JsonSerializer();
                serializer.Formatting = Formatting.Indented;
                serializer.Serialize(streamWriter, items);
            }
        }
    }
}


static void SerializeBigDataViaStream()
{
    BooksList = new List<Book>();
    for (int i = 0; i < 5000000; i++)
    {
        BooksList.Add(new Book()
        {
            Id = i + 1,
            ISBN = $"ISBN_{Guid.NewGuid().ToString("N")}",
            Name = $"Name_{i + 1}",
            Title = $"Title_{i + 1}",
            Topic = $"Topic_{i + 1}"
        });
    }

    string jsonFile = $"{DateTime.Now.ToString("yyyyMMddHHmmssffff")}_{Guid.NewGuid().ToString("N")}.json";
    using(FileStream fs=File.Create(jsonFile))
    {
        fs.WriteJsonToStream<Book> (BooksList);
        MessageBox.Show($"Serialized {BooksList.Count} items!");
    }
}

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text; 
using Newtonsoft.Json;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using System.Windows;
using System.Threading;

namespace ConsoleApp3
{
    internal class Program
    {
        static void Main(string[] args)
        {
            AlarmDelEvent += Program_AlarmDelEvent;
            Thread td = new Thread(() =>
            {
                System.Timers.Timer tmr = new System.Timers.Timer();
                tmr.Interval = 1000;
                tmr.Elapsed += Tmr_Elapsed;
                tmr.Start();
            });
            td.Start();
            //BigDataSerializeRegular();
            SerializeBigDataViaStream();
            Console.ReadLine();
        }

        private static void Tmr_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            var procMem = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64;
            Console.WriteLine(procMem.ToString());
            if (procMem > 4000000000)
            {
                AlarmDelEvent?.Invoke(procMem);
            }
        }

        public static List<Book> BooksList { get; set; }

        delegate void AlarmDel(double d);
        static event AlarmDel AlarmDelEvent;
        static void BigDataSerializeRegular()
        { 
            BooksList = new List<Book>();
            for (int i = 0; i < 5000000; i++)
            {
                BooksList.Add(new Book()
                {
                    Id = i + 1,
                    ISBN = $"ISBN_{Guid.NewGuid().ToString("N")}",
                    Name = $"Name_{i + 1}",
                    Title = $"Title_{i + 1}",
                    Topic = $"Topic_{i + 1}"
                });
            }

            string jsonStr = JsonConvert.SerializeObject(BooksList, Newtonsoft.Json.Formatting.Indented);
            string jsonFile = $"{DateTime.Now.ToString("yyyyMMddHHmmssffff")}_{Guid.NewGuid().ToString("N")}.json";
            using (StreamWriter writer = new StreamWriter(jsonFile, true, Encoding.UTF8))
            {
                writer.WriteLine(jsonStr);
                MessageBox.Show($"Saved {BooksList.Count()} in {jsonFile}");
            }
        }

        static void SerializeBigDataViaStream()
        {
            BooksList = new List<Book>();
            for (int i = 0; i < 15000000; i++)
            {
                BooksList.Add(new Book()
                {
                    Id = i + 1,
                    ISBN = $"ISBN_{Guid.NewGuid().ToString("N")}",
                    Name = $"Name_{i + 1}",
                    Title = $"Title_{i + 1}",
                    Topic = $"Topic_{i + 1}"
                });
            }

            string jsonFile = $"{DateTime.Now.ToString("yyyyMMddHHmmssffff")}_{Guid.NewGuid().ToString("N")}.json";
            using(FileStream fs=File.Create(jsonFile))
            {
                fs.WriteJsonToStream<Book> (BooksList);
                MessageBox.Show($"Serialized {BooksList.Count} items!");
            }
        }

        private static void Program_AlarmDelEvent(double d)
        {
            MessageBox.Show($"Over memory:{d}\n");
        }
    }

    public class Book
    {
        public int Id { get; set; }
        public string ISBN { get; set; }
        public string Name { get; set; }
        public string Title { get; set; }
        public string Topic { get; set; }

    }

    public static class BigDataSerializeStream
    {
        public static void WriteJsonToStream<T>(this Stream stream,IEnumerable<T> items)
        {
            using(var streamWriter=new StreamWriter(stream))
            {
                using(var jsonWriter=new JsonTextWriter(streamWriter))
                {  
                    var serializer = new JsonSerializer();
                    serializer.Formatting = Formatting.Indented;
                    serializer.Serialize(streamWriter, items);
                }
            }
        }
    }
       
}