#!perl

#######################################################
# Skype log splitter
# Version 1.1 dev
# 08.02.2020
#######################################################
 
use strict;
use warnings;

use utf8; 
use feature 'unicode_strings';
use Encode::Locale;
use Encode;

# List of the core modules, which are used
use JSON;
use Data::Dumper;

# Name of the input file with the default value
my $input_file_name = 'messages.json';

# Name and path to the file with table of contents
my $TOC_file_name = '__table_of_contents.txt';

# Name of the output directory
my $output_directory = 'OUTPUT';

# Create the output directory if it doesn't exist
mkdir $output_directory if (!-d $output_directory);

# Open the input file
my $file_handle;
my $input_file_contents;

# Open the file and read its contents
print "\nOpening the input file...\n";
if (-f $input_file_name) {
	if (open ($file_handle, "< :encoding(UTF-8)",$input_file_name)) {
		$input_file_contents = <$file_handle>;
		close($file_handle) or print ("Could not close $input_file_name - $!\n");
	} else {
		print ("Could not open $input_file_name\n$!\n");
		exit 1;
	}
} else {
	# If no input file exist
	print ">>> Input file input_file_name doesn't exist!\n";
	exit 1;
}

# Parse the JSON data from the input file
print "\nProcessing the input file...\n";
my $decoded_content;
eval {$decoded_content = from_json($input_file_contents);};
if ($@) {
	print ">>> ERROR - Unable to parse JSON from the input file\n";
	print "$@\n";
	exit 1;
}

# Save split entries to the output files
print "\nSaving the output files...\n";
my $thread_counter = 0;
my %THREAD_NAMES;
foreach my $current_entry (@{$decoded_content->{'conversations'}}) {
	# Construct the name of the output file
	my $thread_name = '(unknown name)';
	# Try to extract the name of the chat
	if ((exists $current_entry->{'displayName'}) and (defined $current_entry->{'displayName'})) {
		$thread_name = $current_entry->{'displayName'};
	}
	my $output_file_name = 'thread_'.sprintf("%04d",$thread_counter).'.json';
	# Gather extracted thread names for the table of contents
	$THREAD_NAMES{$output_file_name} = $thread_name;
	$output_file_name = $output_directory.'/'.$output_file_name;
	$thread_counter++;
	# Save the contents
	print "> Saving thread: $thread_counter\n";
	open ($file_handle, "> :encoding(UTF-8)", $output_file_name) or print ("Could not open $output_file_name\n$!\n");
	print $file_handle to_json($current_entry);
	close($file_handle) or print ("Could not close $output_file_name - $!\n");
}

# Save the table of contents
print "\nCreating the table of contents...\n";
$TOC_file_name = $output_directory.'/'.$TOC_file_name;
open ($file_handle, "> :encoding(UTF-8)", $TOC_file_name) or print ("Could not open $TOC_file_name\n$!\n");
foreach my $current_entry (sort keys %THREAD_NAMES) {
	print $file_handle "$current_entry      $THREAD_NAMES{$current_entry}\n";
}
close($file_handle) or print ("Could not close $TOC_file_name - $!\n");

#!perl

#######################################################
# Skype log converter
# Version 1.0 dev
# 04.10.2019
# By Q-bit array
# Version 1.1 dev
# 20.01.2021
# By colt_browning
#######################################################
#
# Usage: perl log_converter.pl INPUT_FILE OUTPUT_FILE
#
#######################################################
 
use strict;
use warnings;

use utf8; 
use feature 'unicode_strings';

# List of the core modules, which are used
use JSON;
use Data::Dumper;


# Name of the config file
my $config_file_name = 'log_converter.cfg';

# User aliases and colors for automatic replacements
my %USER_ALIASES;
my %USER_COLORS;

# Skype log file contents
my $input_file_contents;
my $decoded_content;
my @OUTPUT_DATA;


# Map the user skype name to the corresponding Wikipedia name
sub convert_user_name {
	my ($user_name) = @_;
	# Remove the leading prefix
	$user_name =~ s#^8:##;
	# Select the user color from the hash
	if (exists $USER_ALIASES{$user_name}) {
		return $USER_ALIASES{$user_name};
	} else {
		return $user_name.' {{highlight|(!)}}';
	}
}

# Get the color assigned to a given user
sub get_user_color {
	my ($user_name) = @_;
	# Select the user color from the hash
	if (exists $USER_COLORS{$user_name}) {
		return $USER_COLORS{$user_name};
	} else {
		return 'X';
	}
}

# Converts the date to a human-readable format
sub convert_date {
	my ($input_date_string) = @_;
	# Months names
	my @MONTHS = ('января', 'февраля', 'марта', 'апреля', 'мая', 'июня', 'июля', 'августа', 'сентября', 'октября', 'ноября', 'декабря');
	# Parse the input date
	if ($input_date_string =~ m#(\d\d\d\d)-(\d\d)-(\d\d)#) {
		my $year = $1;
		my $month = $2;
		my $day = $3;
		# Get the month name
		my $month_text = $MONTHS[(int $month) - 1];
		return $day.' '.$month_text.' '.$year;
	} else {
		# Unrecognized format - return as is
		return $input_date_string;
	}
}

# Converts UNIX time to a timestamp
# Parameters:
#	UNIX time
# Return value:
#	Timestamp string DD MM YYY HH:MM:SS
sub get_log_timestamp {
	my ($timestamp) = @_;
	# Months names
	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($timestamp);
	my $time_string = sprintf("%02d", $mday).'.'.sprintf("%02d", $mon + 1).'.'.($year+1900).' '.sprintf("%02d", $hour).':'.sprintf("%02d", $min).':'.sprintf("%02d", $sec);
	return $time_string;
}

# Load the configuration file
sub load_configuration {
	my $file_handle;
	my @CONFIG_FILE_CONTENTS;
	print "\nLoading configuration file...\n";
	if (-f $config_file_name) {
		if (open ($file_handle, "< :encoding(UTF-8)", $config_file_name)) {
			@CONFIG_FILE_CONTENTS = <$file_handle>;
			close($file_handle) or print ("Could not close $config_file_name - $!\n");
		} else {
			print ("Could not open $config_file_name\n$!\n");
			exit 1;
		}
	} else {
		# If no input file exist
		print ">>> Input file $config_file_name doesn't exist!\n";
		exit 1;
	}
	# Process the configuration
	foreach my $current_entry (@CONFIG_FILE_CONTENTS) {
		if (length($current_entry) < 10) {
			# Empty line
		} elsif ($current_entry =~ m/^#/) {
			# Comment line
		} elsif ($current_entry =~ m/^(.+?)####(.+?)####(.+?)$/) {
			my $skype_name = $1;
			my $wikipedia_name = $2;
			my $color = $3;
			# Store in the hashes
			$USER_ALIASES{$skype_name} = $wikipedia_name;
			$USER_COLORS{$wikipedia_name} = $color;
		} else {
			# Not supported entry
		}
	}
}

# Open the input file and read its contents
sub read_input_file {
	my ($input_file_name) = @_;
	my $file_handle;
	my $input_file_contents;
	print "\nOpening the input file...\n";
	if (-f $input_file_name) {
		if (open ($file_handle, "< :encoding(UTF-8)",$input_file_name)) {
			$input_file_contents = <$file_handle>;
			close($file_handle) or print ("Could not close $input_file_name - $!\n");
		} else {
			print ("Could not open $input_file_name\n$!\n");
			exit 1;
		}
	} else {
		# If no input file exist
		print ">>> Input file $input_file_name doesn't exist!\n";
		exit 1;
	}
	return $input_file_contents;
}

# Write the results to the output file
sub write_output_file {
	my ($output_file_name) = @_;
	my $file_handle;
	open ($file_handle, "> :encoding(UTF-8)",$output_file_name) or print ("Could not open $output_file_name\n$!\n");
	$"="\n";
	print $file_handle "@OUTPUT_DATA";
	close($file_handle) or print ("Could not close $output_file_name - $!\n");
}

# Parse the JSON data from the input file
sub parse_JSON {
	my ($input_file_contents) = @_;
	print "\nProcessing the input file...\n";
	my $decoded_content;
	eval {$decoded_content = from_json($input_file_contents);};
	if ($@) {
		print ">>> ERROR - Unable to parse JSON from the input file\n";
		print "$@\n";
		exit 1;
	}
	return $decoded_content;
}

# Data processing
sub process_data {
	# Add header
	push (@OUTPUT_DATA, '{{Дискуссия арбитров|XX}}');
	# Messages
	my $last_date = '';
	my $previous_message_body = '';
	# Main entry iteration
	foreach my $current_entry (reverse @{$decoded_content->{'MessageList'}}) {
		# Indicates if the current log entry is empty ( = does not contain any relevant data)
		my $is_empty = 0;
		# print Dumper($current_entry);
		# Skip messages, which were re-sent
		next if (exists $current_entry->{'properties'}{'isserversidegenerated'});
		# Get the interesting parts of the JSON data structure
		my $message_body = $current_entry->{'content'};
		my $full_timestamp = $current_entry->{'originalarrivaltime'};
		my $author =  $current_entry->{'from'};
		# Reprocess the author name
		$author = convert_user_name($author);
		# Reprocess the message body
		# Convert line breaks to <br/> tags
		$message_body =~ s#(\r\n|\n\r|\n|\r)\s*#<br/>#mg;
		# Remove whitespaces in the beginning
		$message_body =~ s#^\s*##mg;
		# Some special character combinations
		$message_body =~ s#(\||\*|==|~~)#<nowiki>$1</nowiki>#mgi;
		# Process pings and convert the name of the pinged user
		my $at = '@';
		$message_body =~ s#<at id="8:(.+?)">.*?</at>#"'''@".convert_user_name($1).":'''"#mgie;
		# Smileys
		$message_body =~ s#<ss type=".+?">(.+?)</ss>#{{highlight|'''$1'''|yellow}}#mg;
		# Entities
		$message_body =~ s#&quot;#"#mg;
		$message_body =~ s#&apos;#'#mg;
		$message_body =~ s#&gt;#<nowiki>></nowiki>#mg;
		$message_body =~ s#&lt;#<nowiki><</nowiki>#mg;
		# Attached files and images
		$message_body =~ s#<URIObject.+?</URIObject>#{{highlight|<<<загруженное изображение>>>}}#mg;
		$message_body =~ s#<MediaAlbum.+?</MediaAlbum>#{{highlight|<<<загруженный файл>>>}}#mg;
		# Text formatting
		$message_body =~ s#<i raw_pre="_" raw_post="_">(.*?)</i>#''$1''#mg;
		$message_body =~ s#<b raw_pre="\*" raw_post="\*">(.*?)</b>#'''$1'''#mg;
		$message_body =~ s#\{<pre raw_pre="\{code\}" raw_post="\{code\}">\}(.*?)\{</pre>\}#<pre>$1</pre>#mg;
		$message_body =~ s#<s[^>]+?>(.*?)</s>#<s>$1</s>#mg;
		$message_body =~ s#<pre raw_pre="\{code\}" raw_post="\{code\}">(.*?)</pre>#<pre>$1</pre>#mg;
		# Parse timestamp
		my $date = 'ERROR';
		my $time = 'ERROR '.$full_timestamp;
		if ($full_timestamp =~ m#^(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)(?:\.\d+)?Z$#) {
			$date = $1;
#			$time = $2;
			$time = "$1 $2";
		}
		# Add date entry if required
#		if ($date ne $last_date) {
#			$last_date = $date;
#			push (@OUTPUT_DATA, '; '.convert_date($date));
#		}
		# Construct output entry
		my $output_message = '';
		# Special entries
		# Text deletions
		if (exists $current_entry->{'properties'}{'deletetime'}) {
			my $deletion_timestamp = get_log_timestamp($current_entry->{'properties'}{'deletetime'});
			$output_message = "{{Chat log del|$time|".get_user_color($author)."|$author|dc=$deletion_timestamp}}";
			push (@OUTPUT_DATA, $output_message);
			next;
		}
		# User removals
		if ($message_body =~ m#<deletemember>.*?<eventtime>\d+</eventtime>.*?<initiator>8:(.+?)</initiator>.*?<target>8:(.+?)</target>.*?</deletemember>#) {
			my $admin = convert_user_name($1);
			my $removed_user = convert_user_name($2);
			my $admin_color = get_user_color($admin);
			if ($admin eq $removed_user) {
				# User left the chat by himself
				$output_message = "{{Chat log oper|".$time."|".$admin_color."|".$admin."|'''покинул чат'''}}";
			} else {
				# User was removed
				$output_message = "{{Chat log oper|".$time."|".$admin_color."|".$admin."|'''удалил ".$removed_user." из этого чата'''}}";
			}
			push (@OUTPUT_DATA, $output_message);
			next;
		}
		# User additions
		if ($message_body =~ m#<addmember>.*?<eventtime>\d+</eventtime>.*?<initiator>8:(.+?)</initiator>.*?<target>8:(.+?)</target>.*?</addmember>#) {
			my $admin = convert_user_name($1);
			my $added_user = convert_user_name($2);
			my $admin_color = get_user_color($admin);
			$output_message = "{{Chat log oper|".$time."|".$admin_color."|".$admin."|'''добавил ".$added_user." к этому чату'''}}";
			push (@OUTPUT_DATA, $output_message);
			next;
		}
		# Detect type and complexity of message
		if ($message_body =~ m#=|(<quote author=)|(<a href=)|(<e_m )#) {
			# Advanced structure
			# Resolve links
			$message_body =~ s#<a href="([^>]+?)">[^<]+?</a>#$1#mgi;
			# Additional metadata
			my $quote_text = '';
			my $quote_metadata = '';
			my $answer_to_quote = '';
			my $change_time = '';
			# Process quotations
			if ($message_body =~ m#<quote author="(.+?)".+?timestamp="(.+?)".*?>(.*?)</quote>#i) {
				my $quote_author = convert_user_name($1);
				my $timestamp = get_log_timestamp($2);
				my $full_body = $3;
				# Tidy the quote body
				$full_body =~ s#<legacyquote>.*?</legacyquote>##mgi;
				$full_body =~ s#<e_m.+?</e_m>|<e_m[^>]+?/>##mgi;
				$full_body =~ s#^Edited previous message: ##m;
				$quote_text = "|q=$full_body";
				# Quote metadata
				$quote_metadata = "|qc='''$quote_author''' $timestamp";
				# Remove quotation from the message body
				$message_body =~ s#<quote.+?</quote>##mgi;
			}
			# Process edited replies
			if ($message_body =~ m#(<e_m.+?</e_m>|<e_m[^>]+?/>)#i) {
				# Try to extract the timestamp
				my $tag_content = $1;
				my $timestamp = '';
				my $timestamp_ms = '';
				# There are two different timestamps, but not always present. Try to grab the best one.
				if ($tag_content =~ m#ts="(\d{10})"#) {
					$timestamp = $1;
				}
				if ($tag_content =~ m#ts_ms="(\d{10})\d{3}"#) {
					$timestamp_ms = $1;
				}
				my $timestamp_parsed = 'ERROR';
				if ($timestamp ne '') {
					$timestamp_parsed = get_log_timestamp($timestamp);
				} elsif ($timestamp_ms ne '') {
					$timestamp_parsed = get_log_timestamp($timestamp_ms);
				}
				$change_time = "|ch=$timestamp_parsed";
				# Remove edit marker from the message body
				$message_body =~ s#<e_m.+?</e_m>|<e_m[^>]+?/>##mgi;
				$message_body =~ s#^(Edited previous message: )#{{highlight|$1}}#m;
			}
			$output_message = "{{Chat log ext|$time|".get_user_color($author)."|$author".$quote_text.$quote_metadata."|text=$message_body".$change_time."}}";
			# Detect empty messages
			$is_empty = 1 if (($message_body eq '') and ($quote_text eq ''));
		} else {
			# Simple structure
			# Detect empty messages
			$is_empty = 1 if ($message_body eq '');
			$output_message = "{{Chat log|$time|".get_user_color($author)."|$author|$message_body}}";
		}
		# DEBUG
		# push (@OUTPUT_DATA, ">>> ID: ".$current_entry->{'id'}.' ### '.$current_entry->{'content'}) if (($message_body eq '') and ($current_entry->{'content'} !~ m#<quote author=#));
		# Ignore duplicates and empty entries
		if ((!$is_empty) and ($message_body ne $previous_message_body)) {
			push (@OUTPUT_DATA, $output_message);
		} else {
			# Don't copy the duplicates or empty entries
		}
		# Save the message body from the current iteration
		$previous_message_body = $message_body;
	}
}

#
# MAIN
#
{
	# Check the command line parameters
	if (@ARGV != 2) {
		print "\nERROR - incorrect parameters!\n";
		print "Usage: perl log_converter.pl INPUT_FILE_NAME OUTPUT_FILE_NAME\n";
		exit 1;
	}
	my ($input_file_name, $output_file_name) = @ARGV;

	# Load the configuration file
	load_configuration();

	# Process the logs
	$input_file_contents = read_input_file($input_file_name);
	$decoded_content = parse_JSON($input_file_contents);
	process_data();
	write_output_file($output_file_name);
}

#########################################################
# Файл конфигурации скрипта для переработки логов скайпа
#########################################################

# Необходим для автоматической замены имён учёток в скайпе на википедийные имена и добавления цвета мантии арбитров
# Формат записи:
# Имя_учётки_в_скайпе####Имя_участника_в_Википедии####Цвет_судейской_мантии

# Закомментировать строку можно с помощью символа '#' в начале строки

# Пример:

dima_carn####Carn####7