Public/tornfarm
1
0
Fork 0
mirror of https://github.com/zebrajr/tornfarm.git synced 2025-12-06 12:20:58 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

poc1.0

Browse Source
This commit is contained in:
Carlos Sousa 2021-08-07 13:24:19 +02:00
commit e13a66daae
23 changed files with 1483 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*storage/
docker-compose.yml

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Carlos Sousa
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

6
Legacy/Torn Farm GUI - PHP and MySQL/config.php Normal file
View File

@ -0,0 +1,6 @@
<?php
$server_name = "localhost";
$sqlusername = "[:DEPLOY]##YOUR_MYSQL_USER##";
$sqlpassword = "[:DEPLOY]##YOUR_MYSQL_PWD##";
$sqldatabase = "[:DEPLOY]##YOUR_MYSQL_DATABASE_NAME##";
?>

223
Legacy/Torn Farm GUI - PHP and MySQL/functions.php Normal file
View File

@ -0,0 +1,223 @@
<?php
include "config.php";
// Function to Display the Filter Options (Minimum-Maximum)
function func_display_filter_options($int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name){
echo "<tr>";
echo "<td align='center' rowspan='2'><input type='text' value='" . $str_maximum_rank . "' name=maximum_rank></td>";
echo "<td align='center' rowspan='2'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_role disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_maximum_level . "' name=maximum_level></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_awards disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_age disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_maximum_playerid . "' name=maximum_playerid></td>";
echo "<td align='center' rowspan='2'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_playername disabled></td>";
echo "<td align='center' rowspan='2'><input type='text' value='" . $str_faction_name . "' name=maximum_faction_name ></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_maximumlife disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_lastaction disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_lastcheck disabled></td>";
echo "</tr>";
echo "<tr>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_rank disabled></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_role disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_minimum_level . "' name=minimum_level></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_awards disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_age disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_minimum_playerid . "' name=minimum_playerid></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_playername disabled></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_factionname disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_maximumlife disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_lastaction disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_lastcheck disabled></td>";
echo "</tr>";
echo "<tr>";
echo "<td align='center' colspan='11'>";
echo "<input type='submit' name='Filter' value='Update Filters'>";
echo "</td>";
echo "</tr>";
}
// Function to Display the Table Headers (Column Names)
function func_display_table_header(){
echo "<tr>";
echo "<td align='center'>Rank</td>";
echo "<td align='center'>Role</td>";
echo "<td align='center'>Level</td>";
echo "<td align='center'>Awards</td>";
echo "<td align='center'>Age</td>";
echo "<td align='center'>PlayerID</td>";
echo "<td align='center'>Player Name</td>";
echo "<td align='center'>Faction Name</td>";
echo "<td align='center'>Maximum Life</td>";
echo "<td align='center'>Last Action</td>";
echo "<td align='center'>Last Check</td>";
echo "</tr>";
}
// Function to Display the Server Stats (Visitors, Indexed players, Last Update etc)
function func_display_server_stats($conn){
// Gets a total of Available IDs
$obj_stmt_targets_count_query = $conn->prepare("SELECT COUNT(ID) FROM torn_list");
$obj_stmt_targets_count_query->execute();
$obj_stmt_targets_count_query->bind_result($obj_stmt_targets_count_query_count);
while($row = $obj_stmt_targets_count_query->fetch()){
$int_count_targets = $obj_stmt_targets_count_query_count;
}
$obj_stmt_targets_count_query->close();
// Gets a total of Ignored IDs
$obj_stmt_ignored_count_query = $conn->prepare("SELECT COUNT(id) FROM torn_list_ignored");
$obj_stmt_ignored_count_query->execute();
$obj_stmt_ignored_count_query->bind_result($obj_stmt_ignored_count_query_count);
while($row = $obj_stmt_ignored_count_query->fetch()){
$int_count_ignored = $obj_stmt_ignored_count_query_count;
}
$obj_stmt_ignored_count_query->close();
// Gets Last time since update tbl.torn_list
$obj_stmt_last_update_query = $conn->prepare("SELECT UPDATE_TIME FROM information_schema.tables WHERE TABLE_SCHEMA = 'carlosso_tornfarm' AND TABLE_NAME = 'torn_list'");
$obj_stmt_last_update_query->execute();
$obj_stmt_last_update_query->bind_result($obj_stmt_last_update_datetime);
while($row = $obj_stmt_last_update_query->fetch()){
$str_last_update_datetime = $obj_stmt_last_update_datetime;
}
$obj_stmt_last_update_query->close();
// Sum Available with Ignored IDs
$int_total_targets = $int_count_targets + $int_count_ignored;
// Displays the information
echo "<tr>";
echo "<td align='center'>Last Update (Frontend): <b>2019-11-08 20:35:00</b></td>";
echo "<td align='center'>Total IDs Indexed: <b>" . $int_total_targets . "</b></td>";
echo "</tr><tr>";
echo "<td align='center'>Last Update (Database): <b>" . $str_last_update_datetime . "</b></td>";
echo "<td align='center'>Total Players Indexed: <b>" . $int_count_targets . "</b></td>";
echo "</tr>";
}
// Function to get the User IP Address
function getUserIP(){
// Get real visitor IP behind CloudFlare network
if (isset($_SERVER["HTTP_CF_CONNECTING_IP"])) {
$_SERVER['REMOTE_ADDR'] = $_SERVER["HTTP_CF_CONNECTING_IP"];
$_SERVER['HTTP_CLIENT_IP'] = $_SERVER["HTTP_CF_CONNECTING_IP"];
}
$client = @$_SERVER['HTTP_CLIENT_IP'];
$forward = @$_SERVER['HTTP_X_FORWARDED_FOR'];
$remote = $_SERVER['REMOTE_ADDR'];
if(filter_var($client, FILTER_VALIDATE_IP))
{ $ip = $client; }
elseif(filter_var($forward, FILTER_VALIDATE_IP))
{ $ip = $forward; }
else { $ip = $remote; }
return $ip;
}
// Function to add IP to the DB if it's a new IP
// [TODO] Add SQL Injection Protection
function func_display_visitors($conn){
$user_ip = getUserIP();
$sql = "SELECT * FROM visitors WHERE IP = " . "'" . $user_ip . "'" ;
$result = $conn->query($sql);
if ($result->num_rows <= 0){
$sql = "INSERT INTO `visitors` (`ip`) VALUE (" . "'" . $user_ip . "'" . ")";
$conn->query($sql);
}
$sql = "SELECT COUNT(ID) FROM visitors";
$result = mysqli_query($conn, $sql);
$row = mysqli_fetch_array($result);
#$result = $conn->query($sql);
#$row = $result->fetch_assoc());
#$visits = $row['count(id)'];
echo $row[0];
}
function func_display_players($array_targets_selection){
// $tr_switch is used to create the effect of a table design
// using a bgcolor for the tr
$tr_switch = false;
foreach($array_targets_selection as $value){
if ($tr_switch == false){
echo "<tr bgcolor='lightgrey'>";
$tr_switch = true;
} else {
echo "<tr>";
$tr_switch = false;
}
echo "<td align='center'>" . $value['rank'] . "</td>";
echo "<td align='center'>" . $value['role'] . "</td>";
echo "<td align='center'>" . $value['level'] . "</td>";
echo "<td align='center'>" . $value['awards'] . "</td>";
echo "<td align='center'>" . $value['age'] . "</td>";
echo "<td align='center'><a href='https://www.torn.com/profiles.php?XID=" . $value['playerid'] . "' target='_blank'>" . $value['playerid'] . "</td>";
echo "<td align='center'><a href='https://www.torn.com/profiles.php?XID=" . $value['playerid'] . "' target='_blank'>" . $value['name'] ."</td>";
echo "<td align='center'>" . $value['faction_name'] . "</td>";
echo "<td align='center'>" . $value['maximum_life'] . "</td>";
echo "<td align='center'>" . $value['last_action'] . "</td>";
echo "<td align='center'>";
if($value['attack_date'] == "0000-00-00"){echo "";} else {echo $value['attack_date'];};
echo "</td>";
echo "</tr>";
}
}
function func_get_targets_selections($conn, $int_minimum_level, $int_maximum_level, $int_limit, $int_results_per_page, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name){
$str_maximum_rank = $str_maximum_rank . '%';
$str_faction_name = '%' . $str_faction_name . '%';
$query = "SELECT rank, role, level, awards, age, playerid, name, faction_name, maximum_life, last_action, attack_date FROM torn_list WHERE role in ('Civilian') ";
$query .= "AND level >= ? AND level <= ? AND rank LIKE ? AND faction_name LIKE ? AND playerid >= ? and playerid <= ? ORDER BY level ASC LIMIT ?, ?";
$obj_stmt_query_players = $conn->prepare($query);
$obj_stmt_query_players->bind_param('iissiiii', $int_minimum_level, $int_maximum_level, $str_maximum_rank, $str_faction_name, $int_minimum_playerid, $int_maximum_playerid, $int_limit, $int_results_per_page);
$obj_stmt_query_players->execute();
$obj_stmt_query_players->bind_result($result_rank, $result_role, $result_level, $result_awards, $result_age, $result_playerid, $result_name, $result_faction_name, $result_maximum_life, $result_last_action, $result_attack_date);
$array_targets_selection = array();
while($row = $obj_stmt_query_players->fetch()){
$array_values = array();
$array_values["rank"] = $result_rank;
$array_values["role"] = $result_role;
$array_values["level"] = $result_level;
$array_values["awards"] = $result_awards;
$array_values["age"] = $result_age;
$array_values["playerid"] = $result_playerid;
$array_values["name"] = $result_name;
$array_values["faction_name"] = $result_faction_name;
$array_values["maximum_life"] = $result_maximum_life;
$array_values["last_action"] = $result_last_action;
$array_values["attack_date"] = $result_attack_date;
$array_targets_selection[] = $array_values;
}
$obj_stmt_query_players->close();
return $array_targets_selection;
}
// Function to Display the Navigation Pages
function func_display_navigation($conn, $int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_results_per_page, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name){
$str_maximum_rank = $str_maximum_rank . '%';
$str_faction_name = '%' . $str_faction_name . '%';
$query = "SELECT count(id) FROM torn_list WHERE role in ('Civilian') AND level >= ? AND level <= ? AND rank LIKE ? AND faction_name LIKE ? AND playerid >= ? AND playerid <= ? ORDER BY level ASC";
$obj_stmt_navigation = $conn->prepare($query);
$obj_stmt_navigation->bind_param('iissii', $int_minimum_level, $int_maximum_level, $str_maximum_rank, $str_faction_name, $int_minimum_playerid, $int_maximum_playerid);
$obj_stmt_navigation->execute();
$obj_stmt_navigation->bind_result($result_id);
while($row = $obj_stmt_navigation->fetch()){
$int_total_result = $result_id;
}
$int_total_pages = ceil($int_total_result / $int_results_per_page);
echo "<table style='border:1px solid;'>";
echo "<tr><td align='center' colspan='30'><b>Navigation Pages</b></td></tr>";
echo "<tr>";
for ($i = 1; $i <= $int_total_pages; $i++){
if (($i % 30) == 1){
echo "</tr><tr>";
}
echo "<td align='center'><input type='submit' name='page' value='" . $i . "'></td>";
}
echo "</tr></table>";
}
?>

8
Legacy/Torn Farm GUI - PHP and MySQL/functions_header.php Normal file
View File

@ -0,0 +1,8 @@
<?php
function func_create_header(){
echo "<header>";
echo "<title>Torn Player Database & Indexer</title>";
echo "<meta name='viewport' content='width=device-width, initial-scale=1.0'>";
echo "</header>";
}
?>

122
Legacy/Torn Farm GUI - PHP and MySQL/index.php Normal file
View File

@ -0,0 +1,122 @@
<?php
echo "<html>";
/*
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
*/
include "config.php";
include "functions_header.php";
include "functions.php";
// Creates Headers
func_create_header();
echo "<body>";
echo "<div style='margin:0 auto;'>";
// Declare variables
$int_results_per_page = 100;
$int_minimum_level = 5;
$int_maximum_level = 100;
$int_minimum_playerid = 1;
$int_maximum_playerid = 3500000;
$int_limit = 0;
$str_faction_name = '%';
$str_maximum_rank = '%';
// Gets the User IP
getUserIP();
// Estabilishes the MySQL connection
$conn = mysqli_connect($server_name, $sqlusername, $sqlpassword, $sqldatabase)
or die('Error connecting to MySQL server.');
// Display Unique Visitors count
echo "<form id='filter' action='index.php' method='post'>";
echo "<table border='0' width='100%>";
echo "<tr>";
echo "<td colspan='3' align='center'>";
echo "Total Unique Visits: ";
func_display_visitors($conn);
echo "</td>";
echo "</tr>";
// Create connection
$conn = mysqli_connect($server_name, $sqlusername, $sqlpassword, $sqldatabase)
or die('Error connecting to MySQL server.');
// Gets the first sort
$sort1 = $_GET["sort1"];
$int_selected_page = $_POST["page"];
$int_post_minimum_level = $_POST["minimum_level"];
$int_post_maximum_level = $_POST["maximum_level"];
$str_post_maximum_rank = $_POST["maximum_rank"];
$int_post_minimum_playerid = $_POST["minimum_playerid"];
$int_post_maximum_playerid = $_POST["maximum_playerid"];
$str_post_faction_name = $_POST["maximum_faction_name"];
// Displays the Server Stats
func_display_server_stats($conn);
echo "</table>";
// Starts a New Table
echo "<table style='border:1px solid;'>";
// SQL SELECT to get Indexed Players
// If the Minimum Level Filter was changed
if (isset($int_post_minimum_level)){
$int_minimum_level = $int_post_minimum_level;
}
// If the Maximum Level Filter was changed
if (isset($int_post_maximum_level)){
$int_maximum_level = $int_post_maximum_level;
}
// If the Maximum Rank was changed
if (isset($str_post_maximum_rank)){
$str_maximum_rank = $str_post_maximum_rank;
}
// If the Minimum PlayerID was changed
if (isset($int_post_minimum_playerid)){
$int_minimum_playerid = $int_post_minimum_playerid;
}
// If the Maximum PlayerID was changed
if (isset($int_post_maximum_playerid)){
$int_maximum_playerid = $int_post_maximum_playerid;
}
// If the Faction Name was changed
if (isset($str_post_faction_name)){
$str_faction_name = $str_post_faction_name;
}
// Handler for the Page Selection. Default to user selected, else to page 1
if (isset($int_selected_page)){
$int_limit = ($int_selected_page-1) * $int_results_per_page;
}
// Displays the Table Header
func_display_table_header();
// Displays Filtering Options
func_display_filter_options($int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name);
// Gets and then Displays the Targets according to the selection (Filters + Pages)
$array_targets_selection = func_get_targets_selections($conn, $int_minimum_level, $int_maximum_level, $int_limit, $int_results_per_page, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name);
func_display_players($array_targets_selection);
echo "</table>";
// Displays the Navigation Pages
func_display_navigation($conn, $int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_results_per_page, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name);
echo "</form>";
$conn->close();
echo "</div>";
echo "</body>";
echo "</html>";
?>

94
Legacy/Torn Farm GUI - PHP and MySQL/mysql_create_tables Normal file
View File

@ -0,0 +1,94 @@
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET AUTOCOMMIT = 0;
START TRANSACTION;
SET time_zone = "+00:00";
CREATE DATABASE IF NOT EXISTS `[:DEPLOY]##YOUR_DATABASE_NAME##` DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
USE `[:DEPLOY]##YOUR_DATABASE_NAME##`;
CREATE TABLE `torn_list` (
`id` bigint(20) NOT NULL COMMENT 'Unique ID',
`rank` text NOT NULL COMMENT 'Rank',
`role` text NOT NULL COMMENT 'Role',
`level` int(11) NOT NULL COMMENT 'Level',
`awards` int(11) NOT NULL COMMENT 'Awards',
`age` int(11) NOT NULL COMMENT 'Age',
`playerid` int(11) NOT NULL COMMENT 'PlayerID',
`name` text NOT NULL COMMENT 'Name',
`faction_name` text NOT NULL COMMENT 'Faction Name',
`maximum_life` int(11) NOT NULL COMMENT 'Maximum Life',
`last_action` text NOT NULL COMMENT 'Last Action',
`attack_date` datetime NOT NULL COMMENT 'Date of last attack',
`attack_level` int(11) NOT NULL COMMENT 'Base Stats when Attacking',
`attack_result` text NOT NULL COMMENT '(W)in or (L)oss'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='List from 2019/07/18 onwards';
CREATE TABLE `torn_list_ignored` (
`id` bigint(20) NOT NULL COMMENT 'Unique ID',
`playerid` int(11) NOT NULL COMMENT 'PlayerID'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='List from 2019/07/18 onwards';
CREATE TABLE `torn_ranks` (
`id` int(11) NOT NULL,
`rankid` int(11) NOT NULL,
`rankname` varchar(64) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
INSERT INTO `torn_ranks` (`id`, `rankid`, `rankname`) VALUES
(1, 1, 'Absolute beginner'),
(2, 2, 'Beginner'),
(3, 3, 'Inexperienced'),
(4, 4, 'Rookie'),
(5, 5, 'Novice'),
(6, 6, 'Below Average'),
(7, 7, 'Average'),
(8, 8, 'Reasonable'),
(9, 9, 'Above Average'),
(10, 10, 'Competent'),
(11, 11, 'Highly competent'),
(12, 12, 'Veteran'),
(13, 13, 'Distinguished'),
(14, 14, 'Highly distinguished'),
(15, 15, 'Professional'),
(16, 16, 'Star'),
(17, 17, 'Master'),
(18, 18, 'Outstanding'),
(19, 19, 'Celebrity'),
(20, 20, 'Supreme'),
(21, 21, 'Idolised'),
(22, 22, 'Champion'),
(23, 23, 'Heroic'),
(24, 24, 'Legendary'),
(25, 25, 'Elite'),
(26, 26, 'Invincible');
CREATE TABLE `visitors` (
`id` bigint(20) NOT NULL COMMENT 'ID',
`ip` text NOT NULL COMMENT 'IP'
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
ALTER TABLE `torn_list`
ADD KEY `id` (`id`);
ALTER TABLE `torn_list_ignored`
ADD KEY `id` (`id`);
ALTER TABLE `torn_ranks`
ADD PRIMARY KEY (`id`);
ALTER TABLE `visitors`
ADD PRIMARY KEY (`id`);
ALTER TABLE `torn_list`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Unique ID';
ALTER TABLE `torn_list_ignored`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Unique ID';
ALTER TABLE `torn_ranks`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT, AUTO_INCREMENT=27;
ALTER TABLE `visitors`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'ID';
COMMIT;

2
Legacy/Torn Farm Scraper - Python/start_torn_farm_cloud.py Normal file
View File

@ -0,0 +1,2 @@
nohup python3 torn_farm_cloud.py > torn_farm.log 2>&1 &
echo $! > torn_pid.txt

123
Legacy/Torn Farm Scraper - Python/torn_farm_cloud.py Normal file
View File

@ -0,0 +1,123 @@
# ---------- CONFIG ----------
# Your API Key
apikey = "[:DEPLOY]##YOUR_API_KEY##"
# Starting ID
startingid = 1
# Ending ID
endingid = 2385000
# Maximal API Requests / Minute
maxrequests = 99
# ------------------------------------------------------
# ---------- Do not edit below here if unsure ----------
# ------------------------------------------------------
# ---------- Imports and Main Link Configuration ----------
import json
import time
import sys
import csv
import os
import random
import requests
import mysql.connector
import datetime
homepage1 = "https://api.torn.com/user/"
homepage2 = "?selections=&key="
# --------------------------------
# ---------- Functions ----------
# --------------------------------
# Prints Message With TimeStamp
def addtolog(message):
localtime = time.asctime(time.localtime(time.time()))
tempstring = localtime + " | " + message
print (tempstring)
# ----------------------------------
# ---------- MAIN PROGRAM ----------
# ----------------------------------
current_request = 1
while 1 == 1:
# Creates the DB Connection
mydb = mysql.connector.connect(
host = '[:DEPLOY]##YOUR_MYSQL_DATABASE_HOST##',
user = '[:DEPLOY]##YOUR_MYSQL_USER##',
password = '[:DEPLOY]##YOUR_MYSQL_PWD##',
database = '[:DEPLOY]##YOUR_MYSQL_DATABASE_NAME##'
)
mycursor = mydb.cursor()
# Gets a random user
playerid = random.randint(startingid,endingid)
# Search for ID as ignored user
sqlquery = "SELECT id, playerid FROM torn_list_ignored WHERE playerid = %s"
sqlid = (playerid, )
mycursor.execute(sqlquery, sqlid)
myresult = mycursor.fetchall()
# Assumes there isn't a Skip Needed
SkipAction = False
# Checks if the ID is known as empty and set as Skip if needed
if ((len(myresult) == 1)):
SkipAction = True
# Checks if the X Calls per Minute have been reached and waits if needed
if ((current_request >= maxrequests) and not (SkipAction)):
addtolog("Max. Requests Reached. Waiting 40 seconds.")
current_request = 1
time.sleep(50)
# Creates Full Link and makes the Web Request
if not SkipAction:
fulllink = homepage1 + str(playerid) + homepage2 + apikey
response = json.loads(requests.get(fulllink).text)
# If it contains an Error Attribute
if ((response.get('error')) and not (SkipAction)):
if (response.get('error').get('code') == 5):
addtolog("Too Many Requests. Waiting 10 seconds...")
current_request = 1
time.sleep(10)
else:
# Error other then 5 means that the PlayerID is invalid
message = [playerid, response.get('error').get('code'), response.get('error').get('error')]
print(message)
# Add the user to the table
sqlquery = "INSERT INTO torn_list_ignored (playerid) VALUES (%s)"
values = (playerid, )
mycursor.execute(sqlquery, values)
mydb.commit()
# In case Data is returned
if ((response.get('rank')) and not (SkipAction)):
# Search for ID as known user
sqlquery = "SELECT id, playerid FROM torn_list WHERE playerid = %s"
sqlid = (playerid, )
mycursor.execute(sqlquery, sqlid)
myresult = mycursor.fetchall()
csv = [playerid, response.get('rank'), response.get('role'), response.get('level'), response.get('awards'), response.get('age'), response.get('player_id'), response.get('name'), response.get('faction').get('faction_name'), response.get('life').get('maximum'), response.get('last_action').get('relative'), datetime.datetime.now(), '0', '-']
# If the User is found
if(len(myresult) == 1):
print ("Updating PlayerID:", playerid)
sqlquery = "UPDATE torn_list SET rank = %s, role = %s, level = %s, awards = %s, age = %s, name = %s, faction_name = %s, maximum_life = %s, last_action = %s, attack_date = %s WHERE playerid = %s"
values = (csv[1], csv[2], csv[3], csv[4], csv[6], csv[7], csv[8], csv[9], csv[10], csv[11], csv[0])
# If the User isn't found
if(len(myresult) == 0):
print ("Creating PlayerID:", playerid)
sqlquery = "INSERT INTO torn_list (rank, role, level, awards, age, playerid, name, faction_name, maximum_life, last_action, attack_date, attack_level, attack_result) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
values = (csv[1], csv[2], csv[3], csv[4], csv[5], csv[6], csv[7], csv[8], csv[9], csv[10], csv[11], csv[12], csv[13])
mycursor.execute(sqlquery, values)
mydb.commit()
if not SkipAction:
current_request += 1
# Closes the DB Connection
mydb.close()

65
README.md Normal file
View File

@ -0,0 +1,65 @@
# tornfarm
To Deploy:
- Clone or make the files deploy.sh.sample and docker-compose.yml
- Rename deploy.sh.sample to deploy.sh and edit to your own information
- Set the deploy.sh to executable (sudo chmod +x deploy.sh) and run it (./deploy.sh)
- Start the container with docker-compose (docker-compose up OR docker-compose up -d)
- Stop the container and edit the newly created settings.conf
- Start the container again
To Build:
- Move to Current (cd Current)
- docker build --tag <your-tag:your-version> .
# IMPORTANT!!!
Only Relevant to the "Legacy" Edition <br />
This Version is Deprecated and will no longer be updated or maintained.<br />
A python script that scraps the TORN Text-Based MMORPG Player Database and indexes it, and a front and backoffice in PHP / MySQL to use that information
If you are deploying this do as following:
- 1) Have Python
(3.8.1) is what I'm using and check the import on the torn_farm_cloud.py.
Ps.: Sorry I forgot to write down which modules I installed with pip, but feel free to open an issue / discussion
and tell me which modules you had to install. It should easy to debug because the import is basically the first
result of the pip repository.
- 2) Have Apache, MySQL and PHPMyAdmin
installed or similar and deploy the database. I Recommenx XAMPP / LAMPP depending
on your OS if you are starting stuff like this.
- 3) Search for [:DEPLOY]
you will find everything that you have to configure to your own lab to have it running.
- File Description
[:TODO] This should be moved to each individually file description.
- config.php Configuration File for the MySQL on the PHP Side
- functions.php Functions and Procedures used in the PHP Files
- functions_header.php Functions and Procedures used in the PHP Headers
- index.php Main PHP File
- mysql_create_tables SQL Syntax to create the required database tables
- start_torn_farm_cloud_py Starts the main scrap and sends the ProcessID to a file so you can kill it if needed
- torn_farm_cloud.py Python Script that Scraps the Torn Player Database for Public Information
Ps.: Feel free to improve :)

41
bot.Dockerfile Normal file
View File

@ -0,0 +1,41 @@
# Use the official image as a parent image
FROM alpine:latest as build
# Set the Workdir
WORKDIR /scrapper
# Copy the requirement file
COPY src/scrapper/ .
# Install Dependencies
RUN apk add --update --no-cache python3 && \
ln -sf python3 /usr/bin/python && \
python3 -m ensurepip && \
pip3 install --no-cache --upgrade pip setuptools && \
ls -la && \
ls -la config/ && \
pwd && \
pip3 install --requirement requirements.txt
#WORKDIR /app
# Copy the script file
#COPY dev/app .
# Labels and information
LABEL maintainer="Zebra <csousa90@gmail.com>" \
org.label-schema.vendor="Zebra" \
org.label-schema.name="TornScrapper" \
org.label-schema.url="www.carlossousa.tech" \
org.label-schema.description="A Torn Scrapper, which will find and index player public information for easier target selection."
# Creates a Single Layer Image
# Info: Not possible currently, since the Software is NOT self contained eg:
#FROM scratch
#WORKDIR /app
#COPY --from=build /app .
#WORKDIR /app
ENTRYPOINT ["python3", "-u", "scrapper.py"]
#CMD ["python3", "-u", "scrapper.py"]

44
docker-compose.yml Normal file
View File

@ -0,0 +1,44 @@
version: "3.7"
services:
web-server:
container_name: webServerBench
build:
dockerfile: front.Dockerfile
context: .
restart: unless-stopped
volumes:
- "./src/front/:/var/www/html/"
ports:
- "1237:80"
torn-scrapper:
container_name: tornScrapper
build:
dockerfile: bot.Dockerfile
context: .
depends_on:
- mysql-server
restart: unless-stopped
volumes:
- "./src/scrapper/:/scrapper"
read_only: true
mysql-server:
container_name: mySqlBench
image: mariadb:10.6
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: secret
volumes:
- "./storage/back/:/var/lib/mysql"
phpmyadmin:
container_name: phpMyAdminBench
image: phpmyadmin/phpmyadmin:5.0.1
restart: unless-stopped
environment:
PMA_HOST: mysql-server
PMA_USER: root
PMA_PASSWORD: secret
ports:
- "1238:80"

46
docker-compose.yml.sample Normal file
View File

@ -0,0 +1,46 @@
version: "3.7"
services:
web-server:
container_name: webServerBench
build:
dockerfile: front.Dockerfile
context: .
restart: unless-stopped
volumes:
- "./src/front/:/var/www/html/"
ports:
- "1237:80"
torn-scrapper:
container_name: tornScrapper
build:
dockerfile: bot.Dockerfile
context: .
depends_on:
- mysql-server
restart: unless-stopped
environment:
- APIKEY="yourApiKey"
volumes:
- "./src/scrapper/:/scrapper"
read_only: true
mysql-server:
container_name: mySqlBench
image: mariadb:10.6
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: secret
volumes:
- "./storage/back/:/var/lib/mysql"
phpmyadmin:
container_name: phpMyAdminBench
image: phpmyadmin/phpmyadmin:5.0.1
restart: unless-stopped
environment:
PMA_HOST: mysql-server
PMA_USER: root
PMA_PASSWORD: secret
ports:
- "1238:80"

5
front.Dockerfile Normal file
View File

@ -0,0 +1,5 @@
FROM php:8.0.7-apache
RUN docker-php-ext-install mysqli pdo pdo_mysql
WORKDIR /var/www/html/
COPY src/front/ .

94
src/back/mysql_create_tables Normal file
View File

@ -0,0 +1,94 @@
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET AUTOCOMMIT = 0;
START TRANSACTION;
SET time_zone = "+00:00";
CREATE DATABASE IF NOT EXISTS `tornFarm` DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
USE `tornFarm`;
CREATE TABLE `torn_list` (
`id` bigint(20) NOT NULL COMMENT 'Unique ID',
`rank` text NOT NULL COMMENT 'Rank',
`role` text NOT NULL COMMENT 'Role',
`level` int(11) NOT NULL COMMENT 'Level',
`awards` int(11) NOT NULL COMMENT 'Awards',
`age` int(11) NOT NULL COMMENT 'Age',
`playerid` int(11) NOT NULL COMMENT 'PlayerID',
`name` text NOT NULL COMMENT 'Name',
`faction_name` text NOT NULL COMMENT 'Faction Name',
`maximum_life` int(11) NOT NULL COMMENT 'Maximum Life',
`last_action` text NOT NULL COMMENT 'Last Action',
`attack_date` datetime NOT NULL COMMENT 'Date of last attack',
`attack_level` int(11) NOT NULL COMMENT 'Base Stats when Attacking',
`attack_result` text NOT NULL COMMENT '(W)in or (L)oss'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='List from 2019/07/18 onwards';
CREATE TABLE `torn_list_ignored` (
`id` bigint(20) NOT NULL COMMENT 'Unique ID',
`playerid` int(11) NOT NULL COMMENT 'PlayerID'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='List from 2019/07/18 onwards';
CREATE TABLE `torn_ranks` (
`id` int(11) NOT NULL,
`rankid` int(11) NOT NULL,
`rankname` varchar(64) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
INSERT INTO `torn_ranks` (`id`, `rankid`, `rankname`) VALUES
(1, 1, 'Absolute beginner'),
(2, 2, 'Beginner'),
(3, 3, 'Inexperienced'),
(4, 4, 'Rookie'),
(5, 5, 'Novice'),
(6, 6, 'Below Average'),
(7, 7, 'Average'),
(8, 8, 'Reasonable'),
(9, 9, 'Above Average'),
(10, 10, 'Competent'),
(11, 11, 'Highly competent'),
(12, 12, 'Veteran'),
(13, 13, 'Distinguished'),
(14, 14, 'Highly distinguished'),
(15, 15, 'Professional'),
(16, 16, 'Star'),
(17, 17, 'Master'),
(18, 18, 'Outstanding'),
(19, 19, 'Celebrity'),
(20, 20, 'Supreme'),
(21, 21, 'Idolised'),
(22, 22, 'Champion'),
(23, 23, 'Heroic'),
(24, 24, 'Legendary'),
(25, 25, 'Elite'),
(26, 26, 'Invincible');
CREATE TABLE `visitors` (
`id` bigint(20) NOT NULL COMMENT 'ID',
`ip` text NOT NULL COMMENT 'IP'
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
ALTER TABLE `torn_list`
ADD KEY `id` (`id`);
ALTER TABLE `torn_list_ignored`
ADD KEY `id` (`id`);
ALTER TABLE `torn_ranks`
ADD PRIMARY KEY (`id`);
ALTER TABLE `visitors`
ADD PRIMARY KEY (`id`);
ALTER TABLE `torn_list`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Unique ID';
ALTER TABLE `torn_list_ignored`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Unique ID';
ALTER TABLE `torn_ranks`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT, AUTO_INCREMENT=27;
ALTER TABLE `visitors`
MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'ID';
COMMIT;

6
src/front/config.php Normal file
View File

@ -0,0 +1,6 @@
<?php
$server_name = "mySqlBench";
$sqlusername = "root";
$sqlpassword = "secret";
$sqldatabase = "tornFarm";
?>

265
src/front/functions.php Normal file
View File

@ -0,0 +1,265 @@
<?php
include "config.php";
// Function to Display the Filter Options (Minimum-Maximum)
function func_display_filter_options($int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax){
# First Row, as in, "Max Values"
echo "<tr>";
echo "<td align='center' rowspan='2'><input type='text' value='" . $str_maximum_rank . "' name=maximum_rank></td>";
echo "<td align='center' rowspan='2'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_role disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_maximum_level . "' name=maximum_level></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_awards disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_age disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_maximum_playerid . "' name=maximum_playerid></td>";
echo "<td align='center' rowspan='2'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_playername disabled></td>";
echo "<td align='center' rowspan='2'><input type='text' value='" . $str_faction_name . "' name=maximum_faction_name ></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_maximumlife disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_lastaction disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_lastcheck disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_totalCrimes disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_networth disabled></td>";
echo "<td align='center'><input type='text' value ='" . $intMaxXanax . "' name=intMaxXanax ></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_energydrinks disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_energyrefills disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=maximum_se disabled></td>";
echo "</tr>";
echo "</tr>";
# Second Row, as in, "Min Values"
echo "<tr>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_rank disabled></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_role disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_minimum_level . "' name=minimum_level></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_awards disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_age disabled></td>";
echo "<td align='center'><input type='text' value='" . $int_minimum_playerid . "' name=minimum_playerid></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_playername disabled></td>";
# echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_factionname disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_maximumlife disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_lastaction disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_lastcheck disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_totalCrimes disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_networth disabled></td>";
echo "<td align='center'><input type='text' value='" . $intMinXanax . "' name=intMinXanax></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_energydrinks disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_energyrefills disabled></td>";
echo "<td align='center'><input type='text' placeholder='Not Implemented (Yet)' name=minimum_se disabled></td>";
echo "</tr>";
echo "<tr>";
echo "<td align='center' colspan='17'>";
echo "<input type='submit' name='Filter' value='Update Filters'>";
echo "</td>";
echo "</tr>";
}
// Function to Display the Table Headers (Column Names)
function func_display_table_header(){
echo "<tr>";
echo "<td align='center'>Rank</td>";
echo "<td align='center'>Role</td>";
echo "<td align='center'>Level</td>";
echo "<td align='center'>Awards</td>";
echo "<td align='center'>Age</td>";
echo "<td align='center'>PlayerID</td>";
echo "<td align='center'>Player Name</td>";
echo "<td align='center'>Faction Name</td>";
echo "<td align='center'>Maximum Life</td>";
echo "<td align='center'>Last Action</td>";
echo "<td align='center'>Last Check</td>";
echo "<td align='center'>Total<br>Crimes</td>";
echo "<td align='center'>Networth<br>(in M.)</td>";
echo "<td align='center'>Xanax<br>Taken</td>";
echo "<td align='center'>Energy<br>Drink<br>Used</td>";
echo "<td align='center'>Energy<br>Refills</td>";
echo "<td align='center'>SE<br>Used</td>";
echo "</tr>";
}
// Function to Display the Server Stats (Visitors, Indexed players, Last Update etc)
function func_display_server_stats($conn){
// Gets a total of Available IDs
$obj_stmt_targets_count_query = $conn->prepare("SELECT COUNT(ID) FROM torn_list");
$obj_stmt_targets_count_query->execute();
$obj_stmt_targets_count_query->bind_result($obj_stmt_targets_count_query_count);
while($row = $obj_stmt_targets_count_query->fetch()){
$int_count_targets = $obj_stmt_targets_count_query_count;
}
$obj_stmt_targets_count_query->close();
// Gets a total of Ignored IDs
$obj_stmt_ignored_count_query = $conn->prepare("SELECT COUNT(id) FROM torn_list_ignored");
$obj_stmt_ignored_count_query->execute();
$obj_stmt_ignored_count_query->bind_result($obj_stmt_ignored_count_query_count);
while($row = $obj_stmt_ignored_count_query->fetch()){
$int_count_ignored = $obj_stmt_ignored_count_query_count;
}
$obj_stmt_ignored_count_query->close();
// Gets Last time since update tbl.torn_list
# [TODO] TableSchema name should be read from config
$obj_stmt_last_update_query = $conn->prepare("SELECT UPDATE_TIME FROM information_schema.tables WHERE TABLE_SCHEMA = 'tornFarm' AND TABLE_NAME = 'torn_list'");
$obj_stmt_last_update_query->execute();
$obj_stmt_last_update_query->bind_result($obj_stmt_last_update_datetime);
while($row = $obj_stmt_last_update_query->fetch()){
$str_last_update_datetime = $obj_stmt_last_update_datetime;
}
$obj_stmt_last_update_query->close();
// Sum Available with Ignored IDs
$int_total_targets = $int_count_targets + $int_count_ignored;
// Displays the information
echo "<tr>";
echo "<td align='center'>Last Update (Frontend): <b>2021-07-15 20:00:00</b></td>";
echo "<td align='center'>Total IDs Indexed: <b>" . $int_total_targets . "</b></td>";
echo "</tr><tr>";
echo "<td align='center'>Last Update (Database): <b>" . $str_last_update_datetime . "</b></td>";
echo "<td align='center'>Total Players Indexed: <b>" . $int_count_targets . "</b></td>";
echo "</tr>";
}
// Function to get the User IP Address
function getUserIP(){
// Get real visitor IP behind CloudFlare network
if (isset($_SERVER["HTTP_CF_CONNECTING_IP"])) {
$_SERVER['REMOTE_ADDR'] = $_SERVER["HTTP_CF_CONNECTING_IP"];
$_SERVER['HTTP_CLIENT_IP'] = $_SERVER["HTTP_CF_CONNECTING_IP"];
}
$client = @$_SERVER['HTTP_CLIENT_IP'];
$forward = @$_SERVER['HTTP_X_FORWARDED_FOR'];
$remote = $_SERVER['REMOTE_ADDR'];
if(filter_var($client, FILTER_VALIDATE_IP))
{ $ip = $client; }
elseif(filter_var($forward, FILTER_VALIDATE_IP))
{ $ip = $forward; }
else { $ip = $remote; }
return $ip;
}
// Function to add IP to the DB if it's a new IP
// [TODO] Add SQL Injection Protection
function func_display_visitors($conn){
$user_ip = getUserIP();
$sql = "SELECT * FROM visitors WHERE IP = " . "'" . $user_ip . "'" ;
$result = $conn->query($sql);
if ($result->num_rows <= 0){
$sql = "INSERT INTO `visitors` (`ip`) VALUE (" . "'" . $user_ip . "'" . ")";
$conn->query($sql);
}
$sql = "SELECT COUNT(ID) FROM visitors";
$result = mysqli_query($conn, $sql);
$row = mysqli_fetch_array($result);
#$result = $conn->query($sql);
#$row = $result->fetch_assoc());
#$visits = $row['count(id)'];
echo $row[0];
}
# Function to display the values from each indexed user in a Table Format
function func_display_players($array_targets_selection){
// $tr_switch is used to create the effect of a table design
// using a bgcolor for the tr
$tr_switch = false;
foreach($array_targets_selection as $value){
# Row Output
# Simple Switch to improve table user readibility
if ($tr_switch == false){
echo "<tr bgcolor='lightgrey'>";
$tr_switch = true;
} else {
echo "<tr>";
$tr_switch = false;
}
# Actual Row information (Player Rank, Level, Name, etc)
echo "<td align='center'>" . $value['rank'] . "</td>";
echo "<td align='center'>" . $value['role'] . "</td>";
echo "<td align='center'>" . $value['level'] . "</td>";
echo "<td align='center'>" . $value['awards'] . "</td>";
echo "<td align='center'>" . $value['age'] . "</td>";
echo "<td align='center'><a href='https://www.torn.com/profiles.php?XID=" . $value['playerid'] . "' target='_blank'>" . $value['playerid'] . "</td>";
echo "<td align='center'><a href='https://www.torn.com/profiles.php?XID=" . $value['playerid'] . "' target='_blank'>" . $value['name'] ."</td>";
echo "<td align='center'>" . $value['faction_name'] . "</td>";
echo "<td align='center'>" . $value['maximum_life'] . "</td>";
echo "<td align='center'>" . $value['last_action'] . "</td>";
echo "<td align='center'>";
if($value['attack_date'] == "0000-00-00"){echo "";} else {echo $value['attack_date'];};
echo "</td>";
echo "<td align='center'>" . $value['totalCrimes'] . "</td>";
echo "<td align='center'>";
echo round(($value['totalNetworth']/1000000), 2);
echo "</td>";
echo "<td align='center'>" . $value['xanTaken'] . "</td>";
echo "<td align='center'>" . $value['energyDrinkUsed'] . "</td>";
echo "<td align='center'>" . $value['energyRefills'] . "</td>";
echo "<td align='center'>" . $value['statEnchancersUsed'] . "</td>";
echo "</tr>";
}
}
# Function to get the values from the SQL Database
# [TODO] Should be converted to sql_procedures
function func_get_targets_selections($conn, $int_minimum_level, $int_maximum_level, $int_limit, $int_results_per_page, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax){
$str_maximum_rank = $str_maximum_rank . '%';
$str_faction_name = '%' . $str_faction_name . '%';
$query = "SELECT rank, role, level, awards, age, playerid, name, faction_name, maximum_life, last_action, attack_date, totalCrimes, totalNetworth, xanTaken, energyDrinkUsed, energyRefills, statEnhancersUsed FROM torn_list WHERE role in ('Civilian') ";
$query .= "AND level >= ? AND level <= ? AND rank LIKE ? AND faction_name LIKE ? AND playerid >= ? and playerid <= ? and xanTaken <= ? and xanTaken >= ? ORDER BY level ASC LIMIT ?, ?";
$obj_stmt_query_players = $conn->prepare($query);
$obj_stmt_query_players->bind_param('iissiiiiii', $int_minimum_level, $int_maximum_level, $str_maximum_rank, $str_faction_name, $int_minimum_playerid, $int_maximum_playerid, $intMaxXanax, $intMinXanax, $int_limit, $int_results_per_page);
$obj_stmt_query_players->execute();
$obj_stmt_query_players->bind_result($result_rank, $result_role, $result_level, $result_awards, $result_age, $result_playerid, $result_name, $result_faction_name, $result_maximum_life, $result_last_action, $result_attack_date, $result_totalCrimes, $result_totalNetworth, $result_xanTaken, $result_energyDrinkUsed, $result_energyRefills, $result_statEnchancersUsed);
$array_targets_selection = array();
while($row = $obj_stmt_query_players->fetch()){
$array_values = array();
$array_values["rank"] = $result_rank;
$array_values["role"] = $result_role;
$array_values["level"] = $result_level;
$array_values["awards"] = $result_awards;
$array_values["age"] = $result_age;
$array_values["playerid"] = $result_playerid;
$array_values["name"] = $result_name;
$array_values["faction_name"] = $result_faction_name;
$array_values["maximum_life"] = $result_maximum_life;
$array_values["last_action"] = $result_last_action;
$array_values["attack_date"] = $result_attack_date;
$array_values["totalCrimes"] = $result_totalCrimes;
$array_values["totalNetworth"] = $result_totalNetworth;
$array_values["xanTaken"] = $result_xanTaken;
$array_values["energyDrinkUsed"] = $result_energyDrinkUsed;
$array_values["energyRefills"] = $result_energyRefills;
$array_values["statEnchancersUsed"] = $result_statEnchancersUsed;
$array_targets_selection[] = $array_values;
}
$obj_stmt_query_players->close();
return $array_targets_selection;
}
// Function to Display the Navigation Pages
function func_display_navigation($conn, $int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_results_per_page, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax){
$str_maximum_rank = $str_maximum_rank . '%';
$str_faction_name = '%' . $str_faction_name . '%';
$query = "SELECT count(id) FROM torn_list WHERE role in ('Civilian') AND level >= ? AND level <= ? AND rank LIKE ? AND faction_name LIKE ? AND playerid >= ? AND playerid <= ? and xanTaken <= ? and xanTaken >= ? ORDER BY level ASC";
$obj_stmt_navigation = $conn->prepare($query);
$obj_stmt_navigation->bind_param('iissiiii', $int_minimum_level, $int_maximum_level, $str_maximum_rank, $str_faction_name, $int_minimum_playerid, $int_maximum_playerid, $intMaxXanax, $intMinXanax);
$obj_stmt_navigation->execute();
$obj_stmt_navigation->bind_result($result_id);
while($row = $obj_stmt_navigation->fetch()){
$int_total_result = $result_id;
}
$int_total_pages = ceil($int_total_result / $int_results_per_page);
echo "<table style='border:1px solid;'>";
echo "<tr><td align='center' colspan='30'><b>Navigation Pages</b></td></tr>";
echo "<tr>";
for ($i = 1; $i <= $int_total_pages; $i++){
if (($i % 30) == 1){
echo "</tr><tr>";
}
echo "<td align='center'><input type='submit' name='page' value='" . $i . "'></td>";
}
echo "</tr></table>";
}
?>

8
src/front/functions_header.php Normal file
View File

@ -0,0 +1,8 @@
<?php
function func_create_header(){
echo "<header>";
echo "<title>Torn Player Database & Indexer</title>";
echo "<meta name='viewport' content='width=device-width, initial-scale=1.0'>";
echo "</header>";
}
?>

139
src/front/index.php Normal file
View File

@ -0,0 +1,139 @@
<?php
error_reporting(E_ERROR | E_PARSE);
echo "<html>";
/*
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
*/
include "config.php";
include "functions_header.php";
include "functions.php";
// Creates Headers
func_create_header();
echo "<body>";
echo "<div style='margin:0 auto;'>";
// Declare variables
$int_results_per_page = 100;
$int_minimum_level = 5;
$int_maximum_level = 100;
$int_minimum_playerid = 1;
$int_maximum_playerid = 3500000;
$int_limit = 0;
$str_faction_name = '%';
$str_maximum_rank = '%';
$intMaxXanax = 999999;
$intMinXanax = 0;
// Gets the User IP
getUserIP();
// Estabilishes the MySQL connection
$conn = mysqli_connect($server_name, $sqlusername, $sqlpassword, $sqldatabase)
or die('Error connecting to MySQL server.');
// Display Unique Visitors count
echo "<form id='filter' action='index.php' method='get'>";
echo "<table border='0' width='100%>";
echo "<tr>";
echo "<td colspan='3' align='center'>";
echo "Total Unique Visits: ";
func_display_visitors($conn);
echo "</td>";
echo "<td>";
echo "If you want, feel free to <a href='https://www.torn.com/profiles.php?XID=2318723'>Donate</a>";
echo "</td>";
echo "</tr>";
// Create connection
$conn = mysqli_connect($server_name, $sqlusername, $sqlpassword, $sqldatabase)
or die('Error connecting to MySQL server.');
// Gets the first sort
$sort1 = $_GET["sort1"];
$int_selected_page = $_GET["page"];
$int_get_minimum_level = $_GET["minimum_level"];
$int_get_maximum_level = $_GET["maximum_level"];
$str_get_maximum_rank = $_GET["maximum_rank"];
$int_get_minimum_playerid = $_GET["minimum_playerid"];
$int_get_maximum_playerid = $_GET["maximum_playerid"];
$str_get_faction_name = $_GET["maximum_faction_name"];
$getIntMaxXanax = $_GET['intMaxXanax'];
$getIntMinXanax = $_GET['intMinXanax'];
// Displays the Server Stats
func_display_server_stats($conn);
echo "</table>";
// Starts a New Table
echo "<table style='border:1px solid;'>";
// SQL SELECT to get Indexed Players
// If the Minimum Level Filter was changed
if (isset($int_get_minimum_level)){
$int_minimum_level = $int_get_minimum_level;
}
// If the Maximum Level Filter was changed
if (isset($int_get_maximum_level)){
$int_maximum_level = $int_get_maximum_level;
}
// If the Maximum Rank was changed
if (isset($str_get_maximum_rank)){
$str_maximum_rank = $str_get_maximum_rank;
}
// If the Minimum PlayerID was changed
if (isset($int_get_minimum_playerid)){
$int_minimum_playerid = $int_get_minimum_playerid;
}
// If the Maximum PlayerID was changed
if (isset($int_get_maximum_playerid)){
$int_maximum_playerid = $int_get_maximum_playerid;
}
// If the Faction Name was changed
if (isset($str_get_faction_name)){
$str_faction_name = $str_get_faction_name;
}
// If the Max Xanax was changed
if (isset($getIntMaxXanax)){
$intMaxXanax = $getIntMaxXanax;
}
// If the Min Xanax was changed
if (isset($getIntMinXanax)){
$intMinXanax = $getIntMinXanax;
}
// Handler for the Page Selection. Default to user selected, else to page 1
if (isset($int_selected_page)){
$int_limit = ($int_selected_page-1) * $int_results_per_page;
}
// Displays the Table Header
func_display_table_header();
// Displays Filtering Options
func_display_filter_options($int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax);
// Gets and then Displays the Targets according to the selection (Filters + Pages)
$array_targets_selection = func_get_targets_selections($conn, $int_minimum_level, $int_maximum_level, $int_limit, $int_results_per_page, $str_maximum_rank, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax);
func_display_players($array_targets_selection);
echo "</table>";
// Displays the Navigation Pages
func_display_navigation($conn, $int_minimum_level, $int_maximum_level, $str_maximum_rank, $int_results_per_page, $int_minimum_playerid, $int_maximum_playerid, $str_faction_name, $intMaxXanax, $intMinXanax);
echo "</form>";
$conn->close();
echo "</div>";
echo "</body>";
echo "</html>";
?>

9
src/scrapper/config/settings.json.sample Normal file
View File

@ -0,0 +1,9 @@
{
"apikey": "<yourapikey>",
"mysql": {
"host": "<host>",
"user": "<user>",
"pwd": "<pwd>",
"database": "<database>"
}
}

11
src/scrapper/rankAnalyser.py Normal file
View File

@ -0,0 +1,11 @@
def rankAnalyser():
# Main Function
def main():
print "I'm shy :< Don't run me directly!"
# Runs main() only if .py is called directly
if __name__ == "__main__":
main()

2
src/scrapper/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
requests
mysql-connector

147
src/scrapper/scrapper.py Normal file
View File

@ -0,0 +1,147 @@
import json
import time
import sys
import csv
import os
import random
import requests
import mysql.connector
import datetime
# ---------- CONFIG ----------
# Your API Key
apikey = str(os.getenv('APIKEY', '0'))
# Starting ID
startingid = 1
# Ending ID
endingid = 2690000
# Maximal API Requests / Minute
maxrequests = 80
# ------------------------------------------------------
# ---------- Do not edit below here if unsure ----------
# ------------------------------------------------------
# ---------- Imports and Main Link Configuration ----------
homepage1 = "https://api.torn.com/user/"
homepage2 = "?selections=profile,personalstats,crimes&key="
# --------------------------------
# ---------- Functions ----------
# --------------------------------
# Prints Message With TimeStamp
def addtolog(message):
localtime = time.asctime(time.localtime(time.time()))
tempstring = localtime + " | " + message
print (tempstring)
# ----------------------------------
# ---------- MAIN PROGRAM ----------
# ----------------------------------
if(apikey == '0'):
print("No ApiKey Given! Terminating")
sys.exit()
current_request = 1
currentPlayerID = random.randint(startingid,endingid)
while currentPlayerID <= endingid:
# Creates the DB Connection
mydb = mysql.connector.connect(
host = 'mySqlBench',
user = 'root',
password = 'secret',
database = 'tornFarm'
)
mycursor = mydb.cursor()
# Search for ID as ignored user
sqlquery = "SELECT id, playerid FROM torn_list_ignored WHERE playerid = %s"
sqlid = (currentPlayerID, )
mycursor.execute(sqlquery, sqlid)
myresult = mycursor.fetchall()
# Assumes there isn't a Skip Needed
SkipAction = False
# Checks if the ID is known as empty and set as Skip if needed
if ((len(myresult) == 1)):
SkipAction = True
# Checks if the X Calls per Minute have been reached and waits if needed
if ((current_request >= maxrequests) and not (SkipAction)):
addtolog("Max. Requests Reached. Waiting 40 seconds.")
current_request = 1
time.sleep(50)
# Creates Full Link and makes the Web Request
if not SkipAction:
fulllink = homepage1 + str(currentPlayerID) + homepage2 + apikey
response = json.loads(requests.get(fulllink).text)
# If it contains an Error Attribute
if ((response.get('error')) and not (SkipAction)):
if (response.get('error').get('code') == 5):
addtolog("Too Many Requests. Waiting 10 seconds...")
current_request = 1
time.sleep(10)
else:
# Error other then 5 means that the PlayerID is invalid
message = [currentPlayerID, response.get('error').get('code'), response.get('error').get('error')]
print(message)
# Add the user to the table
sqlquery = "INSERT INTO torn_list_ignored (playerid) VALUES (%s)"
values = (currentPlayerID, )
mycursor.execute(sqlquery, values)
mydb.commit()
# In case Data is returned
if ((response.get('rank')) and not (SkipAction)):
# Search for ID as known user
sqlquery = "SELECT id, playerid FROM torn_list WHERE playerid = %s"
sqlid = (currentPlayerID, )
mycursor.execute(sqlquery, sqlid)
myresult = mycursor.fetchall()
csv = [currentPlayerID, \
response.get('rank'), \
response.get('role'), \
response.get('level'), \
response.get('awards'), \
response.get('age'), \
response.get('player_id'), \
response.get('name'), \
response.get('faction').get('faction_name'), \
response.get('life').get('maximum'), \
response.get('last_action').get('relative'), \
datetime.datetime.now(), \
response.get('criminalrecord').get('total'), \
response.get('personalstats').get('networth'), \
response.get('personalstats').get('xantaken'), \
response.get('personalstats').get('energydrinkused'), \
response.get('personalstats').get('refills'), \
response.get('personalstats').get('statenhancersused')]
# If the User is found
if(len(myresult) == 1):
print ("Updating PlayerID:", currentPlayerID)
sqlquery = "UPDATE torn_list SET rank = %s, role = %s, level = %s, awards = %s, age = %s, name = %s, faction_name = %s, maximum_life = %s, last_action = %s, attack_date = %s, totalCrimes = %s, totalNetworth = %s, xanTaken = %s, energyDrinkUsed = %s, energyRefills = %s, statEnhancersUsed = %s WHERE playerid = %s"
values = (csv[1], csv[2], csv[3], csv[4], csv[6], csv[7], csv[8], csv[9], csv[10], csv[11], csv[12], csv[13], csv[14], csv[15], csv[16], csv[17], csv[0])
# Added
# totalCrimes = %s, totalNetworth = %s, xanTaken = %s, energyDrinkUsed = %s, energyRefills = %s, statEnhancersUsed = %s
#totalCrimes, totalNetworth, xanTaken, energyDrinkUsed, energyRefills, statEnhancersUsed
# If the User isn't found
if(len(myresult) == 0):
print ("Creating PlayerID:", currentPlayerID)
sqlquery = "INSERT INTO torn_list (rank, role, level, awards, age, playerid, name, faction_name, maximum_life, last_action, attack_date, totalCrimes, totalNetworth, xanTaken, energyDrinkUsed, energyRefills, statEnhancersUsed) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
values = (csv[1], csv[2], csv[3], csv[4], csv[5], csv[6], csv[7], csv[8], csv[9], csv[10], csv[11], csv[12], csv[13], csv[14], csv[15], csv[16], csv[17])
mycursor.execute(sqlquery, values)
mydb.commit()
if not SkipAction:
current_request += 1
# Closes the DB Connection
mydb.close()
currentPlayerID += 1