Blog

Infinite Dependent Drop Down Lists in в Google Sheets (Part 2)

This article is a continuation of Infinite Dependent Dropdowns in Google Sheets.

Changes and additions to the code are described here, namely:

  • Automatic creation of sheets Nome Data_2;
  • Automatic addition of all formulas on sheet Nome;
  • When editing the row of dependent (linked lists) created earlier, the values and formulas to the right of the edited cell are automatically deleted.

function onEdit(e) {

  var row = e.range.getRow();
  var col = e.range.getColumn();
  var list_name = e.source.getActiveSheet().getName();
  var name = e.value;
  var oldName = e.oldValue;
  var sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Home');
  var mask = JSON.stringify(sh.getRange(row, 1, 1, col).getValues()[0]);
  var sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Home');
  var colMax = sh.getLastColumn();
  
  if(list_name === "Home" && name !== oldName && col < colMax) {
    fillColumn(row, col, mask);
  }
}

function onOpen() {
  var ui = SpreadsheetApp.getUi();
  // Or DocumentApp or FormApp.
  ui.createMenu('Custom Menu')
      .addItem('Create sheets', 'createSheets')
      .addToUi();
}


function fillColumn(row, col, mask) {

// clear dataVal and Value
  var sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Home');
  var colMax = sh.getLastColumn();
  sh.getRange(row, col + 1, 1, colMax).clearDataValidations().clearContent();

// find date
  var sd = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data");
  var arrData = sd.getDataRange().getValues();
  var arrData_2 = [];
  arrData_2.push(arrData[0]);
  
  var iMax = arrData.length - 1;
  for(var i=1; i<=iMax; i++) {
    if(JSON.stringify(arrData[i].slice(0, col)) == mask) {
      arrData_2.push(arrData[i]);
    }
  }
   
// clear Data_2
  var sd_2 = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data_2");
  sd_2.getDataRange().clearContent();

// insert data
  sd_2.getRange(1, 1, arrData_2.length, arrData_2[0].length).setValues(arrData_2);
  
// add dataVal
  col++;
  var list = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K'];
  var sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Home');
  sh.getRange(row, col).setDataValidation(SpreadsheetApp.newDataValidation()
  .setAllowInvalid(false)
  .requireValueInRange(sh.getRange('Data_2!$' + list[col - 1] + '$2:$' + list[col - 1] + '$1000'), true)
  .build());
}

function createSheets() {

// is exist Home?
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  var sd = ss.getSheetByName('Data')
  
// create if not exist
  if(!ss.getSheetByName('Home')) {
    ss.insertSheet('Home', 0);
// create Data Val
    var sh = ss.getSheetByName('Home');
    sh.getRange('Home!A2:A20').setDataValidation(SpreadsheetApp.newDataValidation()
      .setAllowInvalid(false)
      .requireValueInRange(sh.getRange('Data!$A$2:$A'), true)
      .build());
    sh.getRange(1, 1, 1, 10).setValues(sd.getRange(1, 1, 1, 10).getValues()).setFontWeight('bold');

  };

// is exist Data_2?
  if(!ss.getSheetByName('Data_2')) {
   
// create if not exist
    var k = ss.getNumSheets();
    ss.insertSheet('Data_2', k + 1);
    var sd_2 = ss.getSheetByName('Data_2');
    sd_2.getRange(1, 1, 1, 10).setValues(sd.getRange(1, 1, 1, 10).getValues()).setFontWeight('bold');
  };
}

Now, in this version of the program, it is possible to automatically create all the sheets of the file necessary for the script to work (including data formatting and validation) by pressing just one(!) button from the user menu.

All you need for this:

  1. create a sheet named "Data"
  2. copy the data needed for linked (dependent) lists into it
  3. copy and paste this script into a file
  4. refresh page
  5. in the appeared custom menu select: Create sheets

You can get more information from this video (RU voice):




ATTENTION!
This article has a continuation: Infinite Dependent Dropdowns in Google Sheets (part 3)





Read more >>

Infinite Dependent Drop Down Lists in в Google Sheets

The article considers the option of creating dependent drop-down lists on the Google Spreadsheet sheet, practically unlimited neither in the number of linked elements, nor in the number of rows on the sheet.

The program script is shown below:

function onEdit(e) {

  let col = e.range.getColumn();
  let list_name = e.source.getActiveSheet().getName();
  let name = e.value;
  
  if(list_name=="Home") {
    fillColumn(col, name);
  }
}

function fillColumn(col, name) {

// find date
  var sd = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data");
  var iMax = sd.getLastRow();
  var col_data = [];
  
  for(var i=2; i<=iMax; i++) {
    var x = sd.getRange(i, col).getValue();
    if(x == name) {
      col_data.push(sd.getRange(i, col+1).getValue());
    }
  }
   
// clear Data_2
  var sd_2 = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data_2");
  sd_2.getRange(2, col+1, sd_2.getLastRow(), sd_2.getLastColumn()).clearContent();


// insert data
  iMax = col_data.length;

  for(var i=2; i<=iMax+1; i++) {
    sd_2.getRange(i, col+1).setValue(col_data.shift());
  }
}

Please note: sheet names must EXACTLY match the names specified in the script!

Main sheet: "Home"

datasheet: "Data",

intermediate data sheet: "Data_2"

These names in the script are highlighted in red, so, if desired, it will not be difficult to find them to replace with the names of your sheets.


This article has a continuation: Infinite Dependent Dropdowns Lists in Google Sheets (Part 2)

Now, in the new version of the program, it is possible to automatically create all the sheets of the file necessary for the script to work (including data formatting and validation) by pressing just one (!) button from the user menu.


More information you can find in this video (RU voice):

Read more >>

Infinite Dropdown Lists: Personal Financial Plan (example # 1)

This article is a continuation of the article Infinite Dependent Dropdown Lists in Google Sheets (part 3) .

Here's an example of how Infinite Linked Lists can be used in a Personal Financial Plan.

In addition to adding several user-friendly functions (automatic insertion of the payment date and the formula for maintaining the balance of receipts and expenses), the script provided contains a number of significant improvements and improvements:

// global variable
const excludedSheets = ['Summary', 'Category'];
const dataSheet = 'Category';

var colShift = 3;
var rowMin = 4;

function onEdit(e) {

  let row = e.range.getRow();
  let col = e.range.getColumn();
  let sheetName = e.source.getActiveSheet().getName();
  if (excludedSheets.includes(sheetName)) return;
  let name = e.value;
  let oldName = e.oldValue;
  let sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
  let mask = JSON.stringify(sh.getRange(row, colShift+1, 1, col-colShift).getValues()[0]);
  let colMax = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(dataSheet).getLastColumn();
  
  if(name !== oldName && row >= rowMin && col < colMax+colShift) {
    insertDateFormulasAndDataValidation(row, col, sheetName)
    fillColumn(row, col, mask, sheetName);
  }
}

function insertDateFormulasAndDataValidation(row, col, sheetName) {

  if (col == colShift+1) {
    let sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
    let dateValue = sh.getRange(row, 2).getValue();
    if (dateValue == '') {
      let today = new Date();
      let timeZone = Session.getScriptTimeZone();
      let todayStr = Utilities.formatDate(today, timeZone, 'dd.MM.yyyy');
      sh.getRange(row, 2).setValue(todayStr);
    };
    let formulaStr = '=I' + (row-1) + '+H' + row + '-C' + row; 
    sh.getRange(row, 9).setFormula(formulaStr);
    let conN = String.fromCharCode(65+colShift) 
    let adress = sheetName + '!' + conN + (row+1) + ':' + conN + (row+3);
    sh.getRange(adress).setDataValidation(SpreadsheetApp.newDataValidation()
      .setAllowInvalid(false)
      .requireValueInRange(sh.getRange(dataSheet+'!$A$2:$A'), true)
      .build());    
  }
}

function tmp() {
  insertDateFormulasAndDataValidation(7, 4, 'Transactions');
}

function onOpen() {
  let ui = SpreadsheetApp.getUi();
  // Or DocumentApp or FormApp.
  ui.createMenu('Custom Menu')
      .addItem('Create sheets', 'createSheets')
      .addToUi();
}

function fillColumn(row, col, mask, sheetName) {

  let col_data = col - colShift;

// clear dataVal and Value
  let sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
  let colMax = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(dataSheet).getLastColumn();
  sh.getRange(row, col + 1, 1, colMax-col_data).clearDataValidations().clearContent();
 
// find date
  let sd = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(dataSheet);
  let arrData = sd.getDataRange().getValues();
  let arrData_2 = [];
  
  let iMax = arrData.length - 1;
  for(let i=1; i<=iMax; i++) {
    if(JSON.stringify(arrData[i].slice(0, col_data)) == mask) {
      arrData_2.push(arrData[i].slice(0, col_data+1));
    }
  }
  arrData_2 = arrData_2.map(item => item.pop())
  let uniqArrDate_2 = arrData_2.filter(uniqValues); 

// add dataVal
  col++;
  sh.getRange(row, col).setDataValidation(SpreadsheetApp.newDataValidation()
  .setAllowInvalid(false)
  .requireValueInList(uniqArrDate_2, true)
  .build());
}

function uniqValues(item, index, arr) {
  return arr.indexOf(item) === index;
}

You can get more information from this video (RU voice!):

Read more >>

Infinite Dependent Dropdown Lists in Google Sheets (Part 3)

This article is a continuation of the article Infinite Dependent Dropdown Lists in Google Sheets (part 2) .

Changes, changes and additions to the code are described here, namely:

  • Automatic sheet creation Nome ;
  • Automatic addition of all formulas on the Nome sheet;
  • When editing a previously created row of dependent lists, the values ​​and formulas to the right of the edited cell are automatically deleted.
  • Ability to shift the table on the Nome sheet to the right and down by an arbitrary number of columns
  • 10/01/2021 Added check for Home and Data sheets in the file

var colShift = 0;

function onEdit(e) {

  let row = e.range.getRow();
  let col = e.range.getColumn();
  let sheetName = e.source.getActiveSheet().getName();
  let name = e.value;
  let oldName = e.oldValue;
  let sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName(sheetName);
  let mask = JSON.stringify(sh.getRange(row, colShift+1, 1, col-colShift).getValues()[0]);
  if (!doesSheetExist('Home', true)) return;
  if (!doesSheetExist('Data', true)) return;

  let colMax = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data").getLastColumn();
  
  if(sheetName === "Home" && name !== oldName && col < colMax+colShift) {
    fillColumn(row, col, mask);
  }
}

function onOpen() {
  let ui = SpreadsheetApp.getUi();
  // Or DocumentApp or FormApp.
  ui.createMenu('Custom Menu')
      .addItem('Create sheets', 'createSheets')
      .addToUi();
}

function fillColumn(row, col, mask) {

  let col_data = col - colShift;

// clear dataVal and Value
  let sh = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Home');
  let colMax = sh.getLastColumn();
  sh.getRange(row, col + 1, 1, colMax).clearDataValidations().clearContent();
 
// find date
  let sd = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data");
  let arrData = sd.getDataRange().getValues();
  let arrData_2 = [];
  
  let iMax = arrData.length - 1;
  for(let i=1; i<=iMax; i++) {
    if(JSON.stringify(arrData[i].slice(0, col_data)) == mask) {
      arrData_2.push(arrData[i].slice(0, col_data+1));
    }
  }
  arrData_2 = arrData_2.map(item => item.pop())
  let uniqArrDate_2 = arrData_2.filter(uniqValues); 

// add dataVal
  col++;
  sh.getRange(row, col).setDataValidation(SpreadsheetApp.newDataValidation()
  .setAllowInvalid(false)
  .requireValueInList(uniqArrDate_2, true)
  .build());
}

function createSheets() {

// is exist Home?
  let ss = SpreadsheetApp.getActiveSpreadsheet();
  let sd;
  if (doesSheetExist('Data', true)) {
    sd = ss.getSheetByName('Data');
  } else {
    return;
  }
  
  
// create if not exist
  if(!ss.getSheetByName('Home')) {
    ss.insertSheet('Home', 0);
// create Data Val
    var sh = ss.getSheetByName('Home');
    sh.getRange('Home!A2:A20').setDataValidation(SpreadsheetApp.newDataValidation()
      .setAllowInvalid(false)
      .requireValueInRange(sh.getRange('Data!$A$2:$A'), true)
      .build());
    sh.getRange(1, 1, 1, 10).setValues(sd.getRange(1, 1, 1, 10).getValues()).setFontWeight('bold');

  };

}

function uniqValues(item, index, arr) {
  return arr.indexOf(item) === index;
}


function doesSheetExist(sheetName, needAlert) {
  let ss = SpreadsheetApp.getActiveSpreadsheet();
  let allSheets = ss.getSheets().map(sheet => sheet.getName())
  if (allSheets.includes(sheetName)) return true;
  if (needAlert) Browser.msgBox(`Error! Could not find sheet "${sheetName}"!`);
  
  return false;
}

This version of the program retains the ability to automatically create all the sheets of the file necessary for the script to work (including formatting and data validation) by pressing just one (!) button from the user menu.

All you need for this:

  1. create sheet named " Data "
  2. copy the data required for dependent lists into it
  3. copy and paste this script into the file
  4. refresh the page
  5. in the appeared custom menu select: Create sheets

You can get more information from this video (RU voice!):

ATTENTION!
This article continues: Infinite Dropdown Lists: Personal Financial Plan (example # 1)

Read more >>

Google API Sheets in Python Reading and Writing Data Using a Service Account

The script below allows you to access Google Spreadsheets files to extract data from the spreadsheet sheets and write new information.

In order for the script to work, the following conditions must be met:

  • Create Service Account: https://console.cloud.google.com/home/dashboard
  • Create a new project, install virtual environment and install the Google API software package (see video below)
  • Create a file in the service account credentials.json
  • Upload the credentials.json file to the project root
  • Create a Google Spreadsheet file and give the bot access to this file (see video below)
  • Copy files main.py and a1range.py and copy the code below


Code for main.py file:

from a1range import A1Range
import os.path
from googleapiclient.discovery import build
from google.oauth2 import service_account

SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SERVICE_ACCOUNT_FILE = os.path.join(BASE_DIR, 'credentials.json')

credentials = service_account.Credentials.from_service_account_file(
        SERVICE_ACCOUNT_FILE, scopes=SCOPES)

# The ID and range of a sample spreadsheet.
SAMPLE_SPREADSHEET_ID = '1KKOM9tpdCKtQmxyKkP_mcEMj1zreHyZ72cLL0ENXSHw'
SAMPLE_RANGE_NAME = 'Sheet1'


service = build('sheets', 'v4', credentials=credentials).spreadsheets().values()

# Call the Sheets API
# result = service.get(spreadsheetId=SAMPLE_SPREADSHEET_ID,
#                      range=SAMPLE_RANGE_NAME).execute()

# data_from_sheet = result.get('values', [])

array = {'values': [[5, 6, None, 100], ['=SUM(A1:A4)', '=SUM(B1:B4)']]}
range_ = A1Range.create_a1range_from_list('Sheet1', 4, 1, array['values']).format()
response = service.update(spreadsheetId=SAMPLE_SPREADSHEET_ID,
                          range=range_,
                          valueInputOption='USER_ENTERED',
                          body=array).execute()

Code for a1range.py file:

from dataclasses import dataclass


@dataclass
class A1Range:
    sheet_name: str
    start_row: int
    start_col: int
    end_row: int
    end_col: int

    def format(self) -> str:
        start = f"{self.col_number_to_letter(self.start_col)}{self.start_row}"
        end = f"{self.col_number_to_letter(self.end_col)}{self.end_row}"
        return f"{self.sheet_name}!{start}:{end}"

    def iter_rows(self):
        return range(self.start_row, self.end_row + 1)

    def iter_cols(self):
        return range(self.start_col, self.end_col + 1)

    @staticmethod
    def col_number_to_letter(j: int) -> str:
        if 1 <= j <= 26:
            return chr(ord('A') + j - 1)
        elif 27 <= j <= 26 * 26:
            first_letter = chr(ord('A') - 1 + (j - 1) // 26)
            second_letter = chr(ord('A') + (j - 1) % 26)
            return first_letter + second_letter
        else:
            raise ValueError(f"Col number is out of range: {j!r}")

    @staticmethod
    def col_letter_to_number(letters: str) -> int:
        letters = letters.upper()
        if len(letters) == 1 and (ord(letters) < ord('A') or ord(letters) > ord('Z')):
            raise ValueError(f"Col letter is out of range: {letters!r}")

        if len(letters) == 2 and (ord(letters[1]) < ord('A') or ord(letters[1]) > ord('Z')):
            raise ValueError(f"The second Col letter is out of range: {letters!r}")

        if len(letters) == 1:
            return ord(letters) - ord('A') + 1
        elif len(letters) == 2:
            return (ord(letters[0]) - ord('A') + 1) * 26 + ord(letters[1]) - ord('A') + 1

    @staticmethod
    def extract_letters(text) -> str:
        only_letters = ''
        for t in text:
            if t.isalpha():
                only_letters += t
        return only_letters

    @staticmethod
    def extract_digits(text) -> str:
        only_digits = ''
        for t in text:
            if t.isdigit():
                only_digits += t
        return only_digits

    @classmethod
    def parse_a1_range(cls, a1: str):
        if "!" in a1 and ":" in a1:
            sheet_name, cell_range = a1.split('!')
            range_start, range_end = cell_range.split(':')
            start_col, start_row = cls.extract_letters(range_start), cls.extract_digits(range_start)
            end_col, end_row = cls.extract_letters(range_end), cls.extract_digits(range_end)
            return cls(
                sheet_name=sheet_name,
                start_col=cls.col_letter_to_number(start_col),
                start_row=int(start_row),
                end_col=cls.col_letter_to_number(end_col),
                end_row=int(end_row),
            )
        else:
            raise ValueError(f'Error! For method "parse_a1_range()" must be full address!')

    @classmethod
    def create_a1range_from_list(cls, sheet_name, from_row, from_col, array):
        """
        The method find coordinates in format A1Notation for a list with data for inserting into a Google Sheet
        :param sheet_name: sheet name in Google Sheet
        :param from_col: the column coordinate of the upper left corner
        :param from_row: the row coordinate of the upper left corner
        :param array: a list with data for inserting in Google Sheet
        :return: coordinates for Google Sheet in format A1Notation
        """
        count_rows = len(array)
        count_cols = 0
        for row in array:
            if len(row) > count_cols:
                count_cols = len(row)
        return cls(
            sheet_name=sheet_name,
            start_col=from_col,
            start_row=from_row,
            end_col=from_col+count_cols-1,
            end_row=from_row+count_rows-1,
        )

You can find more information in this video (RU voice):

Read more >>

Parsing the site using Apps Script does not work. Why?

The method of parsing (scraping), discussed in the article Parsing (scraping) using Google Apps Script is incredibly simple and easy to use. There is no need for complex programs, or high qualifications or special knowledge, or a lot of experience. All code is written directly in the Google Sheets editor, and then placed and run there.

In addition to all this, the script can be run on a schedule at any time in 24/7 mode, and the result can be processed right there and, if necessary, sent by email or a new file with the received and processed data can be created on Google disk.

However, there may be problems here. First of all, this script does not work with sites (more precisely, with pages) where pre-registration is required. Since, to register on the site, you need to create a session and place its code in special files - cookies. However, the standard UrlFetchApp function used in Apps Script to access a website does not allow cookies.

There is another problem, which has recently manifested itself more and more often. This is the unwillingness of site owners to share information with bots. Therefore, you can often see a picture when the html-code issued by the server at the request of the script is significantly different from the code that is issued to the browser.

Actually, precisely in order to quickly find out whether it is possible to parse (scrap) the site we need or not, this program was written:

function onOpen() {
  var ui = SpreadsheetApp.getUi();
  // Or DocumentApp or FormApp.
  ui.createMenu('Custom Menu')
      .addItem('Check', 'fastAnalyzer')
      .addSeparator()
      .addItem('Create Sheeet CHECK', 'createSheet')
      .addToUi();
}


function fastAnalyzer() {
  const ss = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Check');
  const urlDomain = ss.getRange(2, 1).getValue();
  
  let url = urlDomain;
  let count = 0;
  let response = UrlFetchApp.fetch(url);
  let lll = response.getContentText().length;
  ss.getRange(2, 2).setValue(response.getResponseCode());
  ss.getRange(2, 3).setValue(lll);
  let html = response.getContentText();
  let whatLookingFor = ss.getRange(3, 2).getValue();
  let pos = html.indexOf(whatLookingFor);
  ss.getRange(3, 3).setValue(pos);

  ss.getRange(4, 1, ss.getLastRow(), 2).clearContent();
  let totalNumber = html.length
  let i = 0;

  // Check defoult value of delta - number of symbols in 1 row
  // If the cell is empty, print the defole value = 5000
  let delta = Number.parseInt(ss.getRange(2, 4).getValue());
  if (isNaN(delta)) {
    delta = 5000
    // Print defoult value of number of symbols in 1 row
    ss.getRange(2, 4).setValue(delta);
  };

  let iStart = 0;
  while (true) {
    let iEnd = iStart + delta;
    if (iEnd > totalNumber) iEnd = totalNumber; 
    ss.getRange(4 + i, 1).setValue("Символы от " + iStart + " до " + iEnd);
    ss.getRange(4 + i, 2).setValue(html.slice(iStart,iEnd));
    // let currentRow = (4+i) + ":" + (4+i)
    // ss.getRange(3, 3).setValue(currentRow);
    // ss.getRange(currentRow).activate();
    // ss.setRowHeight(4+i, 200);
    i++;
    if (iEnd - iStart < delta) break;
    iStart = iEnd;

  };
}

function createSheet() {

// is exist Check sheet?
  let s = SpreadsheetApp.getActiveSpreadsheet();
  
// create if not exist
  if(!s.getSheetByName('Check')) {
    s.insertSheet('Check', 0);
    ssc = s.getSheetByName('Check');

    ssc.getRange(1, 1).setValue("Url сайта");
    ssc.getRange(1, 2).setValue("Код ответа сайта");
    ssc.getRange(1, 3).setValue("Число символов на выбранной странице");
    ssc.getRange(1, 4).setValue("Изменить вывод числа знаков в однй строке");
    ssc.getRange(2, 1).setBackground('ACCENT3');  // yellow background cell
    ssc.getRange(3, 1).setValue("Какой элемент кода ищем?");
    ssc.getRange(3, 2).setBackground('ACCENT3');  // yellow background cell
  };
}

So, all you need to work with this program is

  • create a new Google Spreadsheet
  • open a code editor in this file
  • completely copy the code from this article into the code editor of the file you just created (and don't forget to save it!)
  • refresh the spreadsheet page and select a custom menu item: Create Sheet CHECK

The program is ready to go! All that is needed now is to add the site url to the Google tables page, the code element that you want to find on this page, and run the script. The answer will be ready in a second!

You can find more information in this video (RU voice):

Read more >>

Web Scraping With Google Apps Script

The script below allows you to automatically find and download information from one of the freelance exchanges.

The domain name is taken from the Google Spread Sheets page. And search results are also uploaded there.

The function scraper() , which contains two cycles, is used as a control script. In the first loop ( for-loop ), we access the web pages of the site and save them in the variable html . In the second loop ( while-loop ), this variable is sequentially processed using three auxiliary functions:

  • The getBlock function finds the part of the html code (code block) inside the tag (usually by the unique value of the attributes of this tag), and returns this block as a string value;
  • The deleteBlock function, on the contrary, deletes the found fragment of the html code inside the block and also returns the remainder of this block as a string value.
  • Unlike the first two functions, the getOpenTag function does not delete the found tag, but returns it as a string value. True, not the whole tag, but only the first (opening part) of this tag.

function scraper() {
  
  const ss = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('Sheet1');
  const urlDomain = ss.getRange(1, 1).getValue();
  
  let url = urlDomain;
  let count = 0;
  for (let page = 1; page < 5; page++) {
    url = urlDomain + page + '/';
    if (page == 1) url = urlDomain;
    let response = UrlFetchApp.fetch(url);
    ss.getRange(2, 1).setValue(response.getResponseCode());
    let html = response.getContentText();
    let p = 0;
    while (true) {
    
      let out = getBlock(html, 'div', html.indexOf('class="JobSearchCard-primary"', p));
      let block = out[0];
      p = out[1] + 1;
      if (p == 0) break;
      
      let title1 = getBlock(block, 'div', 0)[0];
      let title = getBlock(title1, 'a', 0)[0];
      
      let link = getOpenTag(title1, 'a', 0);
      link = getAttrName(link, 'href', 0)
      let formula = '=HYPERLINK("https://www.freelancer.com' +link + '", "' + title + '")';
      ss.getRange(3 + 3 * count, 2).setValue(formula);
      
      let price = getBlock(block, 'div', block.indexOf('class="JobSearchCard-primary-price'))[0];
      if (price.includes('span')) price =  deleteBlock(price, 'span', price.indexOf('span'));
      ss.getRange(3 + 3 * count + 1, 2).setValue(price).setHorizontalAlignment('right');
      
      let description = getBlock(block, 'p', block.indexOf('class="JobSearchCard-primary-description"'))[0];
      ss.getRange(3 + 3 * count, 1, 3).mergeVertically().setValue(description)
          .setBorder(true, true, true, true, null, null, '#000000', SpreadsheetApp.BorderStyle.SOLID)
          .setVerticalAlignment('middle')
          .setWrapStrategy(SpreadsheetApp.WrapStrategy.WRAP);
  
      ss.getRange(3 + 3 * count, 2, 3).setBorder(true, true, true, true, null, null, '#000000', SpreadsheetApp.BorderStyle.SOLID);
      
      let cat = getBlock(block, 'div', block.indexOf('class="JobSearchCard-primary-tags"'))[0];
      cat = cat.split('</a>').map(item => item.split('>')[1]);
      cat.pop();
      cat = cat.join(', ');
      ss.getRange(3 + 3 * count + 2, 2).setValue(cat);
    
      count++;
    
    }; 
  };
}

function getAttrName(html, attr, i) {
  let idxStart = html.indexOf(attr , i);
  if (idxStart == -1) return "Can't to find attr " + attr + ' !';
  idxStart = html.indexOf('"' , idxStart) + 1;
  let idxEnd = html.indexOf('"' , idxStart);
  return html.slice(idxStart,idxEnd).trim();
}

function getOpenTag(html, tag, idxStart) {
  let openTag = '<' + tag;
  let lenOpenTag = openTag.length;
  // where we are?
  if (html.slice(idxStart, idxStart + lenOpenTag) != openTag) {
    idxStart = html.lastIndexOf(openTag, idxStart);
    if (idxStart == -1) return "Can't to find openTag " + openTag + ' !';
  };
  // begin loop after openTag
  let idxEnd = html.indexOf('>', idxStart) + 1;
  if (idxStart == -1) return "Can't to find closing bracket '>' for openTag!";
  return html.slice(idxStart,idxEnd).trim();
}

function deleteBlock(html, tag, idxStart) { // delete opening & closing tag and info between them
  let openTag = '<' + tag;
  let lenOpenTag = openTag.length;
  let closeTag = '</' + tag + '>';
  let lenCloseTag = closeTag.length;
  let countCloseTags = 0;
  let iMax = html.length;
  let idxEnd = 0;
  // where we are?
  if (html.slice(idxStart, idxStart + lenOpenTag) != openTag) {
    idxStart = html.lastIndexOf(openTag, idxStart);
    if (idxStart == -1) return ["Can't to find openTag " + openTag + ' !', -1];
  };
  // begin loop after openTag
  let i = html.indexOf('>') + 1;
  
  while (i <= iMax) {
    i++;
    if (i === iMax) {
      return ['Could not find closing tag for ' + tag, -1];
    };
    let carrentValue = html[i];
    if (html[i] === '<'){
      let closingTag = html.slice(i, i + lenCloseTag);
      let openingTag = html.slice(i, i + lenOpenTag);
      if (html.slice(i, i + lenCloseTag) === closeTag) {
        if (countCloseTags === 0) {
          idxEnd = i + lenCloseTag;
          break;
        } else {
          countCloseTags -= 1;
        };
      } else if (html.slice(i, i + lenOpenTag) === openTag) {
        countCloseTags += 1;
      };
    };
  };
  return (html.slice(0, idxStart) + html.slice(idxEnd, iMax)).trim();
}

function getBlock(html, tag, idxStart) {  // <tag .... > Block </tag>
  let openTag = '<' + tag;
  let lenOpenTag = openTag.length;
  let closeTag = '</' + tag + '>';
  let lenCloseTag = closeTag.length;
  let countCloseTags = 0;
  let iMax = html.length;
  let idxEnd = 0;
  // where we are?
  if (html.slice(idxStart, idxStart + lenOpenTag) != openTag) {
    idxStart = html.lastIndexOf(openTag, idxStart);
    if (idxStart == -1) return ["Can't to find openTag " + openTag + ' !', -1];
  };
  // change start - will start after openTag!
  idxStart = html.indexOf('>', idxStart) + 1;
  let i = idxStart;
  
  while (i <= iMax) {
    i++;
    if (i === iMax) {
      return ['Could not find closing tag for ' + tag, -1];
    };
    let carrentValue = html[i];
    if (html[i] === '<'){
      let closingTag = html.slice(i, i + lenCloseTag);
      let openingTag = html.slice(i, i + lenOpenTag);
      if (html.slice(i, i + lenCloseTag) === closeTag) {
        if (countCloseTags === 0) {
          idxEnd = i - 1;
          break;
        } else {
          countCloseTags -= 1;
        };
      } else if (html.slice(i, i + lenOpenTag) === openTag) {
        countCloseTags += 1;
      };
    };
  };
  return [html.slice(idxStart,idxEnd + 1).trim(), idxEnd];
}

You can find more information in this video (RU voice):

Read more >>

Google Apps Script and JavaScript Arrays. Search methods

The examples below detail JavaScript and Apps Script methods for finding array elements: find() , findIndex () , indexOf() , lastIndexOf() , includes() , every() and some() .

Methods findIndex() , indexOf() and lastIndexOf() find the index of the array that satisfies the specified condition, and the find() method find its value.

The includes() method is convenient for use in the if operator, since it returns true if the item is found, and false if not.

The every() method checks that ALL elements of the array match the condition specified in the callback function, and the some() method checks Whether this condition satisfies at least ONE element of the array.

const arr = [1, 2, 3, 4, 5, 3];
  
// find() returns the VALUE of the FIRST item satisfies condition of the callback function
// OR return <underfined>

  let valueGteater3 = arr.find(item => item > 3);
  console.log(valueGteater3); // 4
  
  let valueGteater5 = arr.find(item => item > 5);
  console.log(valueGteater5); // undefined

// findIndex() returns the INDEX of the FIRST item satisfies condition of the callback function
// OR return <-1>

  let indexItemGteater3 = arr.findIndex(item => item > 3);
  console.log(valueGteater3); // 3
  
  let indexItemGteater5 = arr.findIndex(item => item > 5);
  console.log(valueGteater5); // -1
  
// variant for not first item  
  
  let notFirstItem = arr.find((item, index) => {
    if (item > 3 && index > 3) return true;
  });
  console.log(notFirstItem); // 5
  
// indexOf() returns the INDEX of the FIRST item is equal the specified value
// OR return <-1>

  let valueEqual3 = arr.indexOf(3);
  console.log(valueEqual3); // 2
  
  let valueEqual3Next = arr.indexOf(3, 3);
  console.log(valueEqual3Next); // 5
 
  let valueEqual1 = arr.indexOf(1, 3);
  console.log(valueEqual1); // -1
  
// lastIndexOf() returns the INDEX of the LAST item is equal the specified value
// OR return <-1>

  let valueEqual3Last = arr.lastIndexOf(3);
  console.log(valueEqual3Last); // 5
  
  let valueEqual3NextLast = arr.lastIndexOf(3, 4);
  console.log(valueEqual3NextLast); // 2
 
  let valueEqual5Last = arr.lastIndexOf(5, 3);
  console.log(valueEqual5Last); // -1
  
// const arr = [1, 2, 3, 4, 5, 3];
// includes() returns TRUE is element is icluded in the array, and FALSE is not

  let isIncludes3 = arr.includes(3);
  console.log(isIncludes3); // true
  
  let isIncludes6 = arr.includes(6);
  console.log(isIncludes6); // false
  
// every() returns TRUE if ALL elements satisfy the condition, and FALSE is not

  let allPositive = arr.every(item => item > 0);
  console.log(allPositive); // true
  
  let moreThen1 = arr.every(item => item > 1);
  console.log(moreThen1); // false

// some() returns TRUE if AT LEAST ONE element satisfy the condition, and FALSE is not

  let moreThen4 = arr.some(item => item > 4);
  console.log(moreThen4); // true
  
  let moreThen5 = arr.some(item => item > 5);
  console.log(moreThen5); // false

You can find more information in this video (RU voice):

Read more >>

Google Apps Script and JavaScript Arrays. Method .reduce()

Short description of the method

The .reduce() method, like most other methods using the callback function, is a for loop, inside which the callback function sequentially processes all elements of the array. However, this is the only method that uses another parameter - accumulator, due to which the .reduce() method is often used to calculate the sum of the array.

The .reduce() method can pass 4 parameters to the callback function:

  1. the current value of the sum of the element of the array accumulator
  2. the current value of the array element item
  3. optional parameter index - index of the current element
  4. optional and rarely used parameter array

This method can also be used for more complex calculations. For example, below is a script that creates an array of indexes of the elements of the arr array, which are odd numbers.

const arr = [1, 2, 3, 4, 5];
  
// sum with for loop
  let sumArr = 0;
  for (let i = 0; i < arr.length; i++) {
    sumArr += arr[i];
  };
  console.log(sumArr);
  
// sum with forEach method
  let sumArr_1 = 0;
  arr.forEach(item => sumArr_1 += item);
  console.log(sumArr_1);
  
// sum with reduce method
  let sumArr_2 = arr.reduce((acc, item) => acc + item, 0);
  console.log(sumArr_2);
  
// with initial value
  sumArr_2 = arr.reduce((acc, item, index, array) => {
    console.log(`acc = ${acc}, item = ${item}, index = ${index}`);
    return acc + item;
  }, 0);
  console.log(sumArr_2);
  
// without initial value
  sumArr_2 = arr.reduce((acc, item, index, array) => {
    console.log(`acc = ${acc}, item = ${item}, index = ${index}`);
    return acc + item;
  });
  console.log(sumArr_2);
  
// find indexes items that are odd numbers  
  let newArr = arr.reduce((acc, item, index) => {
    if (item % 2 !== 0) acc.push(index);
    return acc;
  }, []);
  console.log(newArr); // [ 0, 2, 4 ]
  
 // reduceRight
  sumArr_2 = arr.reduceRight((acc, item, index, array) => {
    console.log(`acc = ${acc}, item = ${item}, index = ${index}`);
    return acc + item;
  });
  console.log(sumArr_2);

You can find more information in this video (RU voice):

Read more >>

Creating Arrays of Apps Script. Adding, Deleting and Modifying Array Elements

Arrays in Apps Script can be created in several ways:

  1. Declare the name of the array in the line of code and equate it to a sequence of values taken in square brackets;
  2. Read data from a Google SpreadSheet using the .getValues() method;
  3. Dividing a string using the .split([delimiter]) method (usually a space is used as the delimiter delimiter;
  4. PP 3, when a word is used as a string, and as an divide, an empty string.

function createArray() {
  
// ======== HOW TO CREATE ARRAY? ===========

  // 1.) Write
  var arr = [1, 2, 3];
  
  // 2.) Read from spreadsheet
  var ss = SpreadsheetApp.getActiveSpreadsheet().getActiveSheet();
  var arr1 = ss.getDataRange().getValues(); // [[1.0, 2.0, 3.0]]
  
  // 3.) Using string splitting
  var string = 'I am learning Google Spreadsheet';
  var arr2 = string.split(' '); // [I, am, learning, Google, Spreadsheet]
  
  // 4.) Using string splitting
  var arr3 = 'word'.split(''); // [w, o, r, d]
  
}

There are several ways to add elements to an array:

  1. The simplest one is to equate a specific element of an array (by specifying its index in square brackets) to the desired value.

  2. If the maximum index value is exceeded by more than one, then all elements between the old and new maximum indexes are automatically set to null .

  3. Adding elements to the end of an array using the .push (value) method;
  4. Adding elements to the beginning of the array using the .unshift (value) method;
  5. Deleting an element with maximum index .pop () ;

  6. The value of the deleted element can be assigned to a new variable.

  7. Deleting an element with a null index .shift () ;
  8. Address change of array elements using the .splice (index, hawManyDelete, whatInsert) method;

  9. Where:
    • index - index number from which indexes will be inserted / deleted;
    • hawManyDelete - the number of items to be deleted. If hawManyDelete = 0, there will be no deletion;
    • whatInsert - the value of the element (s) that (s) will be inserted, starting at index number index .
  10. Method of changing the order of indexes in an array from direct to reverse and vice versa .reverse() ;
  11. The method for sorting the elements of the array .sort() . Without a parameter in the form of a function, it is used extremely rarely due to its limitations: it only sorts unicode strings in ascending order.

function addDeleteItems() {

// ================ ADD ITEMS =======================

  // 1.) [index]
  var arr4 = [1, 2, 3];
  arr4[3] = 4; // [1.0, 2.0, 3.0, 4.0]
  arr4[5] = 6; // [1.0, 2.0, 3.0, 4.0, null, 6.0]
  arr4[-1] = 0; // [1.0, 2.0, 3.0, 4.0, null, 6.0]
  
  // 2.) .push(value)
  arr4.push(7); //  [1.0, 2.0, 3.0, 4.0, null, 6.0, 7.0]
  
  // 3.) .unshift(value)
  arr4.unshift(0) // [0.0, 1.0, 2.0, 3.0, 4.0, null, 6.0, 7.0]
  
// ================= DEL ITEMS ======================

  // 4.) .pop()
  var x = arr4.pop(); // [0.0, 1.0, 2.0, 3.0, 4.0, null, 6.0]
  
  // 5.) .shift()
  var x = arr4.shift(); // [1.0, 2.0, 3.0, 4.0, null, 6.0]
  
// ===== ADD (INSERT), DELETE, CAHGE =================

  // 6.) .splice(index, hawManyDelete, whatInsert)
  arr4.splice(4, 0, 5); // [1.0, 2.0, 3.0, 4.0, 5.0, null, 6.0]
  arr4.splice(5, 1);    // [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]

// ============ CHANGE ORDER =========================
  // 7.) .reverse()
  arr4.reverse(); // [6.0, 5.0, 4.0, 3.0, 2.0, 1.0]
  
  // 8.) .sort()
  arr4 = [6.0, 5.0, 404.0, 41.0, 4.0, 3.0, 2.0, 1.0];
  arr4.sort();
}

You can find more information and examples in this video (RU voice):

Read more >>

Tags list

    Apps Script      Arrays Java Script      asynchronous code      asyncio      coroutine      Django      Dropdown List      Drop Shipping      Exceptions      GitHub      Google API      Google Apps Script      Google Docs      Google Drive      Google Sheets      multiprocessing      Parsing      Python      regex      Scraping      ssh      Test Driven Development (TDD)      threading      website monitoring      zip