Skip to content
This repository has been archived by the owner on Dec 25, 2024. It is now read-only.

Commit

Permalink
OpenAccountScraper: support ajax cover download
Browse files Browse the repository at this point in the history 
(requires coverBitmap support for AccountItems)
  • Loading branch information
@johan12345
johan12345 committed Oct 19, 2020
1 parent fd36585 commit ca4b725
Show file tree
Hide file tree
Showing 7 changed files with 99 additions and 82 deletions.
49 changes: 12 additions & 37 deletions opacclient/libopac/src/main/java/de/geeksfactory/opacclient/apis/OpenAccountScraper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import de.geeksfactory.opacclient.i18n.StringProvider;
import de.geeksfactory.opacclient.objects.Account;
import de.geeksfactory.opacclient.objects.AccountData;
import de.geeksfactory.opacclient.objects.AccountItem;
import de.geeksfactory.opacclient.objects.Copy;
import de.geeksfactory.opacclient.objects.DetailedItem;
import de.geeksfactory.opacclient.objects.LentItem;
Expand Down Expand Up @@ -710,6 +709,7 @@ void parse_reservations(AccountData data, Document doc) {
}

private void parseReservationsTable(List<ReservedItem> res, Element pendingTable) {
List<CompletableFuture> futures = new ArrayList<>();
for (Element row : pendingTable.select("tr")) {
if (row.select("th").size() > 0) {
continue;
Expand All @@ -724,14 +724,8 @@ private void parseReservationsTable(List<ReservedItem> res, Element pendingTable
ReservedItem item = new ReservedItem();

if (cols.get(1 + offset).select("img").size() > 0) {
String[] sources =
cols.get(1 + offset).select("img").attr("sources").split("\\|");
for (String s : sources) {
if (s.startsWith("http")) {
item.setCover(s);
break;
}
}
futures.add(assignBestCover(item, getCoverUrlList(
cols.get(1 + offset).select("img[id*=coverView]").first())));
}

item.setTitle(cols.get(2 + offset).text().trim());
Expand All @@ -749,10 +743,12 @@ private void parseReservationsTable(List<ReservedItem> res, Element pendingTable

res.add(item);
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
}

private void parseReadyTable(List<ReservedItem> res, Element readyTable) {
DateTimeFormatter fmt = DateTimeFormat.forPattern("dd.MM.yyyy");
List<CompletableFuture> futures = new ArrayList<>();

Map<String, Integer> colmap = new HashMap<>();
for (Element row : readyTable.select("tr")) {
Expand Down Expand Up @@ -790,7 +786,8 @@ private void parseReadyTable(List<ReservedItem> res, Element readyTable) {

if (colmap.containsKey("cover")) {
Element coverColumn = cols.get(colmap.get("cover"));
parseAccountCover(item, coverColumn);
futures.add(assignBestCover(item,
getCoverUrlList(coverColumn.select("img[id*=coverView]").first())));
}
if (colmap.containsKey("title")) {
Element col = cols.get(colmap.get("title"));
Expand Down Expand Up @@ -822,31 +819,7 @@ private void parseReadyTable(List<ReservedItem> res, Element readyTable) {
item.setStatus("Bereitgestellt");
res.add(item);
}
}

private void parseAccountCover(AccountItem item, Element coverColumn) {
if (coverColumn.select("img").size() > 0) {
String[] sources = coverColumn.select("img").attr("sources").split("\\|");
for (String s : sources) {
if (s.startsWith("http") && !s.contains("/vlb.de") &&
!s.contains("www.buchhandel.de")) {
item.setCover(s);
break;
}
}
if (item.getCover() == null &&
coverColumn.select("img").hasAttr("devsources")) {
String[] devsources =
coverColumn.select("img").attr("devsources").split("\\|");
for (String s : devsources) {
if (s.startsWith("http") && !s.contains("/vlb.de") &&
!s.contains("www.buchhandel.de")) {
item.setCover(s);
break;
}
}
}
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
}

void parse_lent(AccountData data, Document doc, Account account) {
Expand All @@ -869,6 +842,7 @@ void parse_lent(AccountData data, Document doc, Account account) {

private void parseLentTable(Document doc, Account account, List<LentItem> lent, Element table) {
DateTimeFormatter fmt = DateTimeFormat.forPattern("dd.MM.yyyy");
List<CompletableFuture> futures = new ArrayList<>();

Map<String, Integer> colmap = new HashMap<>();
Map<String, LentItem> copyIds = new HashMap<>();
Expand Down Expand Up @@ -914,7 +888,8 @@ private void parseLentTable(Document doc, Account account, List<LentItem> lent,

if (colmap.containsKey("cover")) {
Element coverColumn = cols.get(colmap.get("cover"));
parseAccountCover(item, coverColumn);
futures.add(assignBestCover(item,
getCoverUrlList(coverColumn.select("img[id*=coverView]").first())));
}

row.select(".oclc-module-label").remove();
Expand Down Expand Up @@ -963,7 +938,7 @@ private void parseLentTable(Document doc, Account account, List<LentItem> lent,
lent.add(item);
}


CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
fetchProlongability(copyIds, doc, account);
}

Expand Down
2 changes: 1 addition & 1 deletion opacclient/libopac/src/main/java/de/geeksfactory/opacclient/apis/OpenSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ private class AvailabilityRestInfo {
public String ebookRestUrl;
}

private List<String> getCoverUrlList(Element img) {
protected List<String> getCoverUrlList(Element img) {
String[] parts = img.attr("sources").split("\\|");
// Example: SetSimpleCover|a|https://vlb.de/GetBlob.aspx?strIsbn=9783868511291&amp;
// size=S|a|http://www.buchhandel.de/default.aspx?strframe=titelsuche&amp;
Expand Down
17 changes: 16 additions & 1 deletion opacclient/libopac/src/main/java/de/geeksfactory/opacclient/objects/AccountItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import java.io.Serializable;

public abstract class AccountItem implements Serializable {
public abstract class AccountItem implements Serializable, CoverHolder {
protected long account;
protected String title;
protected String author;
Expand All @@ -12,6 +12,7 @@ public abstract class AccountItem implements Serializable {
protected String status;
protected Long dbId;
protected String cover;
protected byte[] coverBitmap;

/**
* @return The ID of the account this item is associated with
Expand Down Expand Up @@ -146,6 +147,20 @@ public void setCover(String cover) {
this.cover = cover;
}

/**
* @return A cover Bitmap for this item. Optional.
*/
public byte[] getCoverBitmap() {
return coverBitmap;
}

/**
* Set a cover Bitmap for this item. Optional.
*/
public void setCoverBitmap(byte[] cover) {
this.coverBitmap = cover;
}

/**
* Set property using the following keys: LentItem: barcode, returndate, homebranch,
* lendingbranch, prolongurl, renewable, download
Expand Down
12 changes: 9 additions & 3 deletions opacclient/opacapp/src/main/java/de/geeksfactory/opacclient/frontend/AccountFragment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.Handler;
import androidx.preference.PreferenceManager;
import android.text.Html;
import android.text.method.LinkMovementMethod;
import android.view.LayoutInflater;
Expand Down Expand Up @@ -67,6 +66,7 @@
import androidx.core.app.ActivityCompat;
import androidx.core.app.ActivityOptionsCompat;
import androidx.fragment.app.Fragment;
import androidx.preference.PreferenceManager;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import androidx.swiperefreshlayout.widget.SwipeRefreshLayout;
Expand Down Expand Up @@ -1203,10 +1203,16 @@ public void display(final AccountData result, boolean fromcache) {

boolean hideCovers = true;
for (LentItem item : result.getLent()) {
if (item.getMediaType() != null || item.getCover() != null) hideCovers = false;
if (item.getMediaType() != null || item.getCover() != null ||
item.getCoverBitmap() != null) {
hideCovers = false;
}
}
for (ReservedItem item : result.getReservations()) {
if (item.getMediaType() != null || item.getCover() != null) hideCovers = false;
if (item.getMediaType() != null || item.getCover() != null ||
item.getCoverBitmap() != null) {
hideCovers = false;
}
}
lentAdapter.setCoversHidden(hideCovers);
resAdapter.setCoversHidden(hideCovers);
Expand Down
51 changes: 28 additions & 23 deletions ...ent/opacapp/src/main/java/de/geeksfactory/opacclient/frontend/adapter/AccountAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import de.geeksfactory.opacclient.frontend.ResultsAdapter;
import de.geeksfactory.opacclient.i18n.AndroidStringProvider;
import de.geeksfactory.opacclient.objects.AccountItem;
import de.geeksfactory.opacclient.utils.BitmapUtils;
import de.geeksfactory.opacclient.utils.ISBNTools;

public abstract class AccountAdapter<I extends AccountItem, VH extends AccountAdapter.ViewHolder<I>>
Expand Down Expand Up @@ -125,7 +126,7 @@ public void setItem(I item) {
ivMediaType.setVisibility(View.GONE);
ivCover.setVisibility(View.GONE);
} else {
if (item.getCover() != null) {
if (item.getCover() != null || item.getCoverBitmap() != null) {
showCover(item);
} else {
showMediaTypeIcon(item);
Expand All @@ -137,28 +138,32 @@ private void showCover(I item) {
ivCover.setVisibility(View.VISIBLE);
ivMediaType.setVisibility(View.GONE);

Drawable loading = VectorDrawableCompat
.create(context.getResources(), R.drawable.ic_loading, null);
Glide.with(context).using(new ISBNToolsUrlLoader(context))
.load(item.getCover())
.placeholder(loading)
.crossFade()
.listener(new RequestListener<String, GlideDrawable>() {
@Override
public boolean onException(Exception e, String model,
Target<GlideDrawable> target, boolean isFirstResource) {
showMediaTypeIcon(item);
return true;
}

@Override
public boolean onResourceReady(GlideDrawable resource, String model,
Target<GlideDrawable> target, boolean isFromMemoryCache,
boolean isFirstResource) {
return false;
}
})
.into(ivCover);
if (item.getCoverBitmap() != null) {
ivCover.setImageBitmap(BitmapUtils.bitmapFromBytes(item.getCoverBitmap()));
} else {
Drawable loading = VectorDrawableCompat
.create(context.getResources(), R.drawable.ic_loading, null);
Glide.with(context).using(new ISBNToolsUrlLoader(context))
.load(item.getCover())
.placeholder(loading)
.crossFade()
.listener(new RequestListener<String, GlideDrawable>() {
@Override
public boolean onException(Exception e, String model,
Target<GlideDrawable> target, boolean isFirstResource) {
showMediaTypeIcon(item);
return true;
}

@Override
public boolean onResourceReady(GlideDrawable resource, String model,
Target<GlideDrawable> target, boolean isFromMemoryCache,
boolean isFirstResource) {
return false;
}
})
.into(ivCover);
}
}

private void showMediaTypeIcon(I item) {
Expand Down
10 changes: 10 additions & 0 deletions opacclient/opacapp/src/main/java/de/geeksfactory/opacclient/storage/AccountDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ private void setAccountItemAttributes(Cursor cursor, AccountItem item) {
String mediatype = cursor.getString(cursor.getColumnIndex("mediatype"));
item.setMediaType(mediatype != null ? SearchResult.MediaType.valueOf(mediatype) : null);
item.setCover(cursor.getString(cursor.getColumnIndex("cover")));
item.setCoverBitmap(cursor.getBlob(cursor.getColumnIndex("coverBitmap")));
}

private ContentValues lentItemToContentValues(LentItem item, long accountId) {
Expand Down Expand Up @@ -291,6 +292,7 @@ private void setAccountItemAttributes(AccountItem item, ContentValues cv, long a
putOrNull(cv, "itemid", item.getId());
putOrNull(cv, "status", item.getStatus());
putOrNull(cv, "cover", item.getCover());
putOrNull(cv, "coverBitmap", item.getCoverBitmap());
putOrNull(cv, "mediatype",
item.getMediaType() != null ? item.getMediaType().toString() : null);
}
Expand All @@ -311,6 +313,14 @@ private void putOrNull(ContentValues cv, String key, String value) {
}
}

private void putOrNull(ContentValues cv, String key, byte[] value) {
if (value != null) {
cv.put(key, value);
} else {
cv.putNull(key);
}
}

public void invalidateCachedData() {
database.delete(AccountDatabase.TABLENAME_LENT, null, null);
database.delete(AccountDatabase.TABLENAME_RESERVATION, null, null);
Expand Down
40 changes: 23 additions & 17 deletions opacclient/opacapp/src/main/java/de/geeksfactory/opacclient/storage/AccountDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,21 @@
* Copyright (C) 2013 by Raphael Michel under the MIT license:
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the Software
* of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the Software
* is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
package de.geeksfactory.opacclient.storage;
Expand All @@ -37,16 +37,17 @@ public class AccountDatabase extends SQLiteOpenHelper {
// CHANGE THIS
public static final String[] COLUMNS_LENT = {"id", "account", "title", "author", "format",
"itemid", "status", "barcode", "deadline", "homebranch", "lending_branch",
"prolong_data", "renewable", "download_data", "ebook", "mediatype", "cover"};
"prolong_data", "renewable", "download_data", "ebook", "mediatype", "cover",
"coverBitmap"};
public static final String[] COLUMNS_RESERVATIONS = {"id", "account", "title", "author",
"format", "itemid", "status", "ready", "expiration", "branch", "cancel_data",
"booking_data", "mediatype", "cover"};
"booking_data", "mediatype", "cover", "coverBitmap"};
public static final String TABLENAME_ACCOUNTS = "accounts";
public static final String TABLENAME_LENT = "accountdata_lent";
public static final String TABLENAME_RESERVATION = "accountdata_reservations";
public static final String TABLENAME_ALARMS = "alarms";
private static final String DATABASE_NAME = "accounts.db";
private static final int DATABASE_VERSION = 28; // REPLACE ONUPGRADE IF YOU
private static final int DATABASE_VERSION = 29; // REPLACE ONUPGRADE IF YOU

private AccountDatabase(Context context) {
super(context, DATABASE_NAME, null, DATABASE_VERSION);
Expand All @@ -63,19 +64,20 @@ public void onCreate(SQLiteDatabase db) {
+ "accounts ( id integer primary key autoincrement,"
+ " bib text," + " label text," + " name text,"
+ " password text," + " cached integer," + " pendingFees text,"
+ " validUntil text," + " warning text," + " passwordValid integer," + " supportPolicyHintSeen integer" + ");");
+ " validUntil text," + " warning text," + " passwordValid integer," +
" supportPolicyHintSeen integer" + ");");
db.execSQL(
"create table " + "accountdata_lent (" + "id integer primary key autoincrement," +
"account integer," + "title text," + "author text," + "format text," +
"itemid text," + "status text," + "barcode text," + "deadline text," +
"homebranch text," + "lending_branch text," + "prolong_data text," +
"renewable integer," + "download_data text," + "ebook integer," +
"mediatype text," + "cover text" + ");");
"mediatype text," + "cover text," + "coverBitmap blob" + ");");
db.execSQL("create table " + "accountdata_reservations (" +
"id integer primary key autoincrement," + "account integer," + "title text," +
"author text," + "format text," + "itemid text," + "status text," + "ready text," +
"expiration text," + "branch text," + "cancel_data text," + "booking_data text," +
"mediatype text," + "cover text" + ");");
"mediatype text," + "cover text," + "coverBitmap blob" + ");");
db.execSQL("create table " + "alarms (" + "id integer primary key autoincrement," +
"deadline text," + "media text," + "alarm text," + "notified integer," +
"finished integer" + ");");
Expand Down Expand Up @@ -219,6 +221,10 @@ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion < 28) {
db.execSQL("alter table accounts add column supportPolicyHintSeen integer");
}
if (oldVersion < 29) {
db.execSQL("alter table accountdata_lent add column coverBitmap blob");
db.execSQL("alter table accountdata_reservations add column coverBitmap blob");
}
}

}

0 comments on commit ca4b725

Please sign in to comment.